nvmeof: add nvme-of gateway crd support
adds CephNVMeOFGateway crd for managing ceph nvme-of gateways.
supports configurable instances, placement, resources,
and networking options.
Signed-off-by: Oded Viner <oviner@redhat.com>
(cherry picked from commit 88a9bf5c7f)
This commit is contained in:
Oded Viner authored and 
Mergify committed
1 parent
ec7572dfcb
commit
8e1122ecfa
37 files changed
+6474
-7
No files matched your search
@@ -29,6 +29,7 @@
|
||||
"monitoring",
|
||||
"multus",
|
||||
"network",
|
||||
"nvmeof",
|
||||
"nfs",
|
||||
"object",
|
||||
"operator",
|
||||
|
||||
@@ -2803,7 +2803,7 @@ CIDRList
|
||||
<h3 id="ceph.rook.io/v1.Annotations">Annotations
|
||||
(<code>map[string]string</code> alias)</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>, <a href="#ceph.rook.io/v1.RGWServiceSpec">RGWServiceSpec</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>, <a href="#ceph.rook.io/v1.RGWServiceSpec">RGWServiceSpec</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>Annotations are annotations</p>
|
||||
@@ -4377,6 +4377,244 @@ string
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.CephNVMeOFGateway">CephNVMeOFGateway
|
||||
</h3>
|
||||
<div>
|
||||
<p>CephNVMeOFGateway represents a Ceph NVMe-oF Gateway</p>
|
||||
</div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<code>metadata</code><br/>
|
||||
<em>
|
||||
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#objectmeta-v1-meta">
|
||||
Kubernetes meta/v1.ObjectMeta
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
Refer to the Kubernetes API documentation for the fields of the
|
||||
<code>metadata</code> field.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>spec</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">
|
||||
NVMeOFGatewaySpec
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<br/>
|
||||
<br/>
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<code>image</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>instances</code><br/>
|
||||
<em>
|
||||
int
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>The number of active gateway instances</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>pool</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Pool is the RADOS pool where NVMe-oF configuration is stored</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>group</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Group is the gateway group name for high availability (ANA group)</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>configMapRef</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
|
||||
If not specified, a default configuration will be generated</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>nvmeofConfig</code><br/>
|
||||
<em>
|
||||
map[string]map[string]string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
|
||||
This allows users to override or add configuration options without needing to manage a ConfigMap</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>placement</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Placement">
|
||||
Placement
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The affinity to place the gateway pods</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>annotations</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Annotations">
|
||||
Annotations
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The annotations-related configuration to add/set on each Pod related object.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>labels</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Labels">
|
||||
Labels
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The labels-related configuration to add/set on each Pod related object.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>resources</code><br/>
|
||||
<em>
|
||||
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core">
|
||||
Kubernetes core/v1.ResourceRequirements
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Resources set resource requests and limits</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>priorityClassName</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>PriorityClassName sets the priority class on the pods</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>hostNetwork</code><br/>
|
||||
<em>
|
||||
bool
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>ports</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.NVMeOFGatewayPorts">
|
||||
NVMeOFGatewayPorts
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Ports configuration for the NVMe-oF gateway</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>livenessProbe</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.ProbeSpec">
|
||||
ProbeSpec
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>A liveness-probe to verify that gateway has valid run-time state.
|
||||
If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>status</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.NVMeOFGatewayStatus">
|
||||
NVMeOFGatewayStatus
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.CephNetworkType">CephNetworkType
|
||||
(<code>string</code> alias)</h3>
|
||||
<div>
|
||||
@@ -8799,7 +9037,7 @@ int
|
||||
<h3 id="ceph.rook.io/v1.Labels">Labels
|
||||
(<code>map[string]string</code> alias)</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>Labels are label for a given daemons</p>
|
||||
@@ -8815,7 +9053,7 @@ int
|
||||
<h3 id="ceph.rook.io/v1.LocalCephxStatus">LocalCephxStatus
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemStatus">CephFilesystemStatus</a>, <a href="#ceph.rook.io/v1.FileMirrorStatus">FileMirrorStatus</a>, <a href="#ceph.rook.io/v1.NFSStatus">NFSStatus</a>, <a href="#ceph.rook.io/v1.ObjectStoreStatus">ObjectStoreStatus</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephFilesystemStatus">CephFilesystemStatus</a>, <a href="#ceph.rook.io/v1.FileMirrorStatus">FileMirrorStatus</a>, <a href="#ceph.rook.io/v1.NFSStatus">NFSStatus</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewayStatus">NVMeOFGatewayStatus</a>, <a href="#ceph.rook.io/v1.ObjectStoreStatus">ObjectStoreStatus</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
|
||||
</p>
|
||||
<div>
|
||||
</div>
|
||||
@@ -10207,6 +10445,315 @@ LocalCephxStatus
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.NVMeOFGatewayPorts">NVMeOFGatewayPorts
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway</p>
|
||||
</div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<code>ioPort</code><br/>
|
||||
<em>
|
||||
int32
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>IOPort is the port for NVMe-oF IO traffic (default: 4420)</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>gatewayPort</code><br/>
|
||||
<em>
|
||||
int32
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>GatewayPort is the port for the gateway service (default: 5500)</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>monitorPort</code><br/>
|
||||
<em>
|
||||
int32
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>MonitorPort is the port for the monitor service (default: 5499)</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>discoveryPort</code><br/>
|
||||
<em>
|
||||
int32
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>DiscoveryPort is the port for discovery service (default: 8009)</p>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephNVMeOFGateway">CephNVMeOFGateway</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway</p>
|
||||
</div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<code>image</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>instances</code><br/>
|
||||
<em>
|
||||
int
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>The number of active gateway instances</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>pool</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Pool is the RADOS pool where NVMe-oF configuration is stored</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>group</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>Group is the gateway group name for high availability (ANA group)</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>configMapRef</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
|
||||
If not specified, a default configuration will be generated</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>nvmeofConfig</code><br/>
|
||||
<em>
|
||||
map[string]map[string]string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
|
||||
This allows users to override or add configuration options without needing to manage a ConfigMap</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>placement</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Placement">
|
||||
Placement
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The affinity to place the gateway pods</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>annotations</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Annotations">
|
||||
Annotations
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The annotations-related configuration to add/set on each Pod related object.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>labels</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Labels">
|
||||
Labels
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>The labels-related configuration to add/set on each Pod related object.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>resources</code><br/>
|
||||
<em>
|
||||
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core">
|
||||
Kubernetes core/v1.ResourceRequirements
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Resources set resource requests and limits</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>priorityClassName</code><br/>
|
||||
<em>
|
||||
string
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>PriorityClassName sets the priority class on the pods</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>hostNetwork</code><br/>
|
||||
<em>
|
||||
bool
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>ports</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.NVMeOFGatewayPorts">
|
||||
NVMeOFGatewayPorts
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>Ports configuration for the NVMe-oF gateway</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>livenessProbe</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.ProbeSpec">
|
||||
ProbeSpec
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>A liveness-probe to verify that gateway has valid run-time state.
|
||||
If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.</p>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.NVMeOFGatewayStatus">NVMeOFGatewayStatus
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephNVMeOFGateway">CephNVMeOFGateway</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway</p>
|
||||
</div>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<code>Status</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.Status">
|
||||
Status
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
(Members of <code>Status</code> are embedded into this type.)
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>cephx</code><br/>
|
||||
<em>
|
||||
<a href="#ceph.rook.io/v1.LocalCephxStatus">
|
||||
LocalCephxStatus
|
||||
</a>
|
||||
</em>
|
||||
</td>
|
||||
<td>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<h3 id="ceph.rook.io/v1.NamedBlockPoolSpec">NamedBlockPoolSpec
|
||||
</h3>
|
||||
<p>
|
||||
@@ -12299,7 +12846,7 @@ string
|
||||
<h3 id="ceph.rook.io/v1.Placement">Placement
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephCOSIDriverSpec">CephCOSIDriverSpec</a>, <a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>, <a href="#ceph.rook.io/v1.StorageClassDeviceSet">StorageClassDeviceSet</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephCOSIDriverSpec">CephCOSIDriverSpec</a>, <a href="#ceph.rook.io/v1.FilesystemMirroringSpec">FilesystemMirroringSpec</a>, <a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec</a>, <a href="#ceph.rook.io/v1.RBDMirroringSpec">RBDMirroringSpec</a>, <a href="#ceph.rook.io/v1.StorageClassDeviceSet">StorageClassDeviceSet</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>Placement is the placement for an object</p>
|
||||
@@ -12722,7 +13269,7 @@ string
|
||||
<h3 id="ceph.rook.io/v1.ProbeSpec">ProbeSpec
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.ObjectHealthCheckSpec">ObjectHealthCheckSpec</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.GaneshaServerSpec">GaneshaServerSpec</a>, <a href="#ceph.rook.io/v1.MetadataServerSpec">MetadataServerSpec</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewaySpec">NVMeOFGatewaySpec</a>, <a href="#ceph.rook.io/v1.ObjectHealthCheckSpec">ObjectHealthCheckSpec</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon</p>
|
||||
@@ -14216,7 +14763,7 @@ int
|
||||
<h3 id="ceph.rook.io/v1.Status">Status
|
||||
</h3>
|
||||
<p>
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephBucketNotification">CephBucketNotification</a>, <a href="#ceph.rook.io/v1.CephObjectRealm">CephObjectRealm</a>, <a href="#ceph.rook.io/v1.CephObjectZone">CephObjectZone</a>, <a href="#ceph.rook.io/v1.CephObjectZoneGroup">CephObjectZoneGroup</a>, <a href="#ceph.rook.io/v1.FileMirrorStatus">FileMirrorStatus</a>, <a href="#ceph.rook.io/v1.NFSStatus">NFSStatus</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
|
||||
(<em>Appears on:</em><a href="#ceph.rook.io/v1.CephBucketNotification">CephBucketNotification</a>, <a href="#ceph.rook.io/v1.CephObjectRealm">CephObjectRealm</a>, <a href="#ceph.rook.io/v1.CephObjectZone">CephObjectZone</a>, <a href="#ceph.rook.io/v1.CephObjectZoneGroup">CephObjectZoneGroup</a>, <a href="#ceph.rook.io/v1.FileMirrorStatus">FileMirrorStatus</a>, <a href="#ceph.rook.io/v1.NFSStatus">NFSStatus</a>, <a href="#ceph.rook.io/v1.NVMeOFGatewayStatus">NVMeOFGatewayStatus</a>, <a href="#ceph.rook.io/v1.RBDMirrorStatus">RBDMirrorStatus</a>)
|
||||
</p>
|
||||
<div>
|
||||
<p>Status represents the status of an object</p>
|
||||
|
||||
@@ -0,0 +1,462 @@
|
||||
---
|
||||
title: NVMe-oF Block Storage
|
||||
---
|
||||
# NVMe-oF Block Storage
|
||||
|
||||
**This feature is experimental**
|
||||
|
||||
NVMe over Fabrics (NVMe-oF) allows RBD volumes to be exposed and accessed via the NVMe/TCP protocol. This enables both Kubernetes pods within the cluster and external clients outside the cluster to connect to Ceph block storage using standard NVMe-oF initiators, providing high-performance block storage access over the network.
|
||||
|
||||
## Goals
|
||||
|
||||
The NVMe-oF integration in Rook serves two primary purposes:
|
||||
|
||||
1. **External Client Access**: Rook serves as a backend for external clients outside the cluster, enabling non-Kubernetes workloads to access Ceph block storage through standard NVMe-oF initiators. This allows organizations to leverage their Ceph storage infrastructure for both containerized and traditional workloads.
|
||||
|
||||
2. **In-Cluster Consumption**: Pods inside the Kubernetes cluster can consume storage via the NVMe-oF protocol, providing an alternative to traditional RBD mounts with potential performance benefits for certain workloads.
|
||||
|
||||
Both use cases are supported, allowing you to choose the appropriate access method based on your specific requirements and deployment scenarios.
|
||||
|
||||
For more background and design details, see the [NVMe-oF gateway design doc](https://github.com/rook/rook/blob/master/design/ceph/ceph-nvmeof-gateway.md).
|
||||
For the Ceph-CSI NVMe-oF design proposal, see the [ceph-csi NVMe-oF proposal](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
This guide assumes a Rook cluster as explained in the [Quickstart Guide](../../Getting-Started/quickstart.md).
|
||||
|
||||
### Requirements
|
||||
|
||||
- **Ceph Version**: Ceph v20 (Tentacle) or later
|
||||
- **Disable the Ceph CSI operator**: We are still updating the Ceph CSI operator with NVMe-oF support. Currently, it is required to disable the CSI operator to test NVMe-oF. In operator.yaml, set `ROOK_USE_CSI_OPERATOR: "false"`.
|
||||
|
||||
## Step 1: Create a Ceph Block Pool
|
||||
|
||||
Before creating the NVMe-oF gateway, you need to create a CephBlockPool that will be used by the gateway:
|
||||
|
||||
```yaml
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: nvmeof
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 3
|
||||
```
|
||||
|
||||
Create the pool:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/nvmeof-pool.yaml
|
||||
```
|
||||
|
||||
## Step 2: Create the NVMe-oF Gateway
|
||||
|
||||
The `CephNVMeOFGateway` CRD manages the NVMe-oF gateway infrastructure. The operator will automatically create the following resources:
|
||||
|
||||
- **Service**: One per gateway instance for service discovery
|
||||
- **Deployment**: One per gateway instance running the NVMe-oF gateway daemon
|
||||
|
||||
Create the gateway:
|
||||
|
||||
```yaml
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephNVMeOFGateway
|
||||
metadata:
|
||||
name: nvmeof
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
# Container image for the NVMe-oF gateway daemon
|
||||
image: quay.io/ceph/nvmeof:1.5
|
||||
# Pool name that will be used by the NVMe-oF gateway
|
||||
pool: nvmeof
|
||||
# ANA (Asymmetric Namespace Access) group name
|
||||
group: group-a
|
||||
# Number of gateway instances to run
|
||||
instances: 1
|
||||
hostNetwork: false
|
||||
```
|
||||
|
||||
Apply the gateway configuration:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/nvmeof-test.yaml
|
||||
```
|
||||
|
||||
Verify the gateway is running:
|
||||
|
||||
```console
|
||||
kubectl get pod -n rook-ceph -l app=rook-ceph-nvmeof
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
rook-ceph-nvmeof-nvmeof-a-85844ff6b8-4r8gj 1/1 Running 0 91s
|
||||
```
|
||||
|
||||
## Step 4: Deploy the NVMe-oF CSI Driver
|
||||
|
||||
The NVMe-oF CSI driver handles dynamic provisioning of volumes. Deploy the CSI provisioner with the NVMe-oF driver.
|
||||
|
||||
Deploy the NVMe-oF CSI provisioner from the example manifest:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/provisioner.yaml
|
||||
```
|
||||
|
||||
Verify the CSI provisioner pod is running:
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=csi-nvmeofplugin-provisioner
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
csi-nvmeofplugin-provisioner-65b4fbbc8-jjsqj 4/4 Running 0 75s
|
||||
```
|
||||
|
||||
## Step 5: Create the StorageClass
|
||||
|
||||
Create a StorageClass that uses the NVMe-oF CSI driver. You'll need to gather the following information from the gateway:
|
||||
|
||||
1. **nvmeofGatewayAddress**: A stable address for the gateway management API
|
||||
2. **nvmeofGatewayPort**: The gateway port (default: 5500)
|
||||
3. **listeners**: A JSON array containing listener information for each gateway instance
|
||||
|
||||
Discover the values to use in the StorageClass:
|
||||
|
||||
1. **nvmeofGatewayAddress**: Use the Service `CLUSTER-IP`.
|
||||
|
||||
```console
|
||||
kubectl get service -n rook-ceph rook-ceph-nvmeof-nvmeof-a
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-nvmeof-nvmeof-a ClusterIP 10.106.98.71 <none> 4420/TCP,5500/TCP,5499/TCP,8009/TCP 24m
|
||||
```
|
||||
|
||||
2. **listeners.address**: Use the gateway pod IP.
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=rook-ceph-nvmeof -o wide
|
||||
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
|
||||
rook-ceph-nvmeof-nvmeof-a-5fd6cd4d46-mrbwk 1/1 Running 0 26m 10.244.0.16 minikube <none> <none>
|
||||
```
|
||||
|
||||
3. **listeners.hostname**: Use the gateway deployment name.
|
||||
|
||||
```console
|
||||
kubectl get deployments.apps -n rook-ceph -l app=rook-ceph-nvmeof
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY UP-TO-DATE AVAILABLE AGE
|
||||
rook-ceph-nvmeof-nvmeof-a 1/1 1 1 27m
|
||||
```
|
||||
|
||||
Create the StorageClass:
|
||||
|
||||
|
||||
```yaml
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-nvmeof
|
||||
parameters:
|
||||
clusterID: rook-ceph
|
||||
pool: nvmeof
|
||||
subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
|
||||
# Management API - talks to gateway to create subsystems/namespaces
|
||||
nvmeofGatewayAddress: "10.106.98.71"
|
||||
nvmeofGatewayPort: "5500"
|
||||
# Data Plane - worker nodes connect here for actual I/O
|
||||
# List ALL gateway pods for HA and multipath
|
||||
listeners: |
|
||||
[
|
||||
{
|
||||
"address": "10.244.0.16",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-a"
|
||||
}
|
||||
]
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
|
||||
provisioner: nvmeof.csi.ceph.com
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
allowVolumeExpansion: false
|
||||
```
|
||||
|
||||
Create the StorageClass:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/storageclass.yaml
|
||||
```
|
||||
|
||||
## Step 6: Create a PersistentVolumeClaim
|
||||
|
||||
Create a PVC using the NVMe-oF storage class:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: nvmeof-external-volume
|
||||
namespace: default
|
||||
spec:
|
||||
storageClassName: ceph-nvmeof
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 128Mi
|
||||
```
|
||||
|
||||
!!! note
|
||||
This PVC is created for CSI driver provisioning. The volume will be accessible via NVMe-oF protocol by both Kubernetes pods within the cluster and external clients outside the cluster using standard NVMe-oF initiators.
|
||||
|
||||
Create the PVC:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/pvc.yaml
|
||||
```
|
||||
|
||||
Verify the PVC is bound:
|
||||
|
||||
```console
|
||||
kubectl get pvc nvmeof-external-volume
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
|
||||
nvmeof-external-volume Bound pvc-b4108580-5cfa-46d3-beff-320088a5bf3c 128Mi RWO ceph-nvmeof 20m
|
||||
```
|
||||
|
||||
## Step 7: Deploy the NVMe-oF CSI Node Plugin
|
||||
|
||||
Deploy the NVMe-oF CSI node plugin:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/node-plugin.yaml
|
||||
```
|
||||
|
||||
Verify the node plugin pod is running:
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=nvmeof.csi.ceph.com-nodeplugin
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
nvmeof.csi.ceph.com-nodeplugin-xnm82 2/2 Running 0 31h
|
||||
```
|
||||
|
||||
## Step 8: Accessing Volumes via NVMe-oF
|
||||
|
||||
Once the PVC is created and bound, the volume is available via NVMe-oF. The volume can be accessed by both Kubernetes pods within the cluster and external clients outside the cluster.
|
||||
|
||||
### Access from Kubernetes Pods
|
||||
|
||||
Kubernetes pods can consume NVMe-oF volumes by mounting the PVC directly. The CSI driver handles the NVMe-oF connection automatically when the pod mounts the volume.
|
||||
|
||||
Create a sample pod that mounts the PVC:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/pod.yaml
|
||||
```
|
||||
|
||||
Verify the pod is running:
|
||||
|
||||
```console
|
||||
kubectl get pods -n default
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
nvmeof-test-pod 1/1 Running 0 29h
|
||||
```
|
||||
|
||||
### Access from External Clients
|
||||
|
||||
External clients outside the Kubernetes cluster can connect to the gateway using standard NVMe-oF procedures.
|
||||
|
||||
#### Prerequisites for External Clients
|
||||
|
||||
- **NVMe-oF Initiator**: The client must have the `nvme-tcp` kernel module loaded and `nvme-cli` installed
|
||||
- **Network Access**: The client must be able to reach the gateway service IP and ports
|
||||
|
||||
#### Discover Subsystems
|
||||
|
||||
From the external client, discover available NVMe-oF subsystems:
|
||||
|
||||
```bash
|
||||
nvme discover -t tcp -a <gateway-service-ip> -s 5500
|
||||
```
|
||||
|
||||
Replace `<gateway-service-ip>` with the gateway service ClusterIP or an accessible endpoint.
|
||||
|
||||
#### Connect to Subsystem
|
||||
|
||||
Connect to the discovered subsystem:
|
||||
|
||||
```bash
|
||||
nvme connect -t tcp -n <subsystem-nqn> -a <gateway-ip> -s 5500
|
||||
```
|
||||
|
||||
Replace:
|
||||
|
||||
- `<subsystem-nqn>` with the `subsystemNQN` value from your StorageClass (e.g., `nqn.2016-06.io.spdk:cnode1.rook-ceph`)
|
||||
- `<gateway-ip>` with the gateway service IP or pod IP
|
||||
|
||||
#### Access the Volume
|
||||
|
||||
Once connected, the NVMe namespace will appear as a block device on the client:
|
||||
|
||||
```bash
|
||||
lsblk | grep nvme
|
||||
```
|
||||
|
||||
The device will typically appear as `/dev/nvmeXnY` where X is the controller number and Y is the namespace ID.
|
||||
|
||||
#### Format and Mount (Optional)
|
||||
|
||||
If you want to format and mount the device:
|
||||
|
||||
```bash
|
||||
# Format the device
|
||||
sudo mkfs.ext4 /dev/nvmeXnY
|
||||
|
||||
# Mount the device
|
||||
sudo mkdir /mnt/nvmeof
|
||||
sudo mount /dev/nvmeXnY /mnt/nvmeof
|
||||
```
|
||||
|
||||
## High Availability
|
||||
|
||||
For production deployments, configure multiple gateway instances for high availability:
|
||||
|
||||
1. **Increase Gateway Instances**: Set `instances: 2` or higher in the `CephNVMeOFGateway` spec
|
||||
2. **Update StorageClass Listeners**: Add all gateway instance addresses and instance names to the `listeners` array
|
||||
3. **Load Balancing**: Each gateway instance has its own Service; list all of them to support multipath/HA
|
||||
|
||||
Example with multiple instances:
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
instances: 2
|
||||
# ... other settings
|
||||
```
|
||||
|
||||
Then update the StorageClass `listeners` to include all gateway instances/services:
|
||||
|
||||
```yaml
|
||||
listeners: |
|
||||
[
|
||||
{
|
||||
"address": "10.99.212.218",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-a"
|
||||
},
|
||||
{
|
||||
"address": "10.99.212.219",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-b"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Check Gateway Pod Logs
|
||||
|
||||
```console
|
||||
kubectl logs -n rook-ceph -l app=rook-ceph-nvmeof --tail=100
|
||||
```
|
||||
|
||||
### Check CSI Provisioner Logs
|
||||
|
||||
```console
|
||||
kubectl logs -n rook-ceph -l app=csi-nvmeofplugin-provisioner --tail=100
|
||||
```
|
||||
|
||||
### Verify Gateway Service
|
||||
|
||||
```console
|
||||
kubectl describe service -n rook-ceph rook-ceph-nvmeof-my-nvmeof-0
|
||||
```
|
||||
|
||||
### Check PVC Events
|
||||
|
||||
```console
|
||||
kubectl describe pvc nvmeof-external-volume
|
||||
```
|
||||
|
||||
### Verify Ceph CSI Config
|
||||
|
||||
Ensure the `rook-ceph-csi-config` ConfigMap exists and contains the cluster configuration:
|
||||
|
||||
```console
|
||||
kubectl get configmap -n rook-ceph rook-ceph-csi-config -o yaml
|
||||
```
|
||||
|
||||
## Teardown
|
||||
|
||||
!!! warning
|
||||
Deleting the PVC will also delete the underlying RBD image and NVMe namespace. Ensure you have backups if needed.
|
||||
|
||||
To clean up all the artifacts created:
|
||||
|
||||
```console
|
||||
# Delete the test pod
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/pod.yaml
|
||||
|
||||
# Delete the PVC
|
||||
kubectl delete pvc nvmeof-external-volume
|
||||
|
||||
# Delete the StorageClass
|
||||
kubectl delete storageclass ceph-nvmeof
|
||||
|
||||
# Delete the NVMe-oF CSI node plugin
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/node-plugin.yaml
|
||||
|
||||
# Delete the NVMe-oF CSI provisioner (includes ServiceAccount/RBAC)
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/provisioner.yaml
|
||||
|
||||
# Delete the NVMe-oF gateway
|
||||
kubectl delete -n rook-ceph cephnvmeofgateway.ceph.rook.io my-nvmeof
|
||||
|
||||
# Delete the block pool (optional)
|
||||
kubectl delete -n rook-ceph cephblockpool.ceph.rook.io nvmeof
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- [Ceph NVMe-oF Documentation](https://docs.ceph.com/en/latest/rbd/nvmeof-overview/)
|
||||
- [Ceph NVMe-oF Initiator Guide](https://docs.ceph.com/en/latest/rbd/nvmeof-initiator-linux/)
|
||||
- [Ceph CSI NVMe-oF Support](https://github.com/ceph/ceph-csi/blob/devel/docs/design/proposals/nvme-of.md)
|
||||
@@ -65,6 +65,7 @@ rules:
|
||||
- cephblockpools
|
||||
- cephfilesystems
|
||||
- cephnfses
|
||||
- cephnvmeofgateways
|
||||
- cephobjectstores
|
||||
- cephobjectstoreusers
|
||||
- cephobjectrealms
|
||||
|
||||
@@ -62,4 +62,14 @@ metadata:
|
||||
labels:
|
||||
{{- include "library.rook-ceph.labels" . | nindent 4 }}
|
||||
{{- include "library.imagePullSecrets" . | nindent 0 }}
|
||||
---
|
||||
# Service account for NVMe-oF gateway
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: rook-ceph-nvmeof
|
||||
namespace: {{ .Release.Namespace }} # namespace:cluster
|
||||
labels:
|
||||
{{- include "library.rook-ceph.labels" . | nindent 4 }}
|
||||
{{- include "library.imagePullSecrets" . | nindent 0 }}
|
||||
{{- end }}
|
||||
@@ -143,6 +143,7 @@ rules:
|
||||
- cephblockpools
|
||||
- cephfilesystems
|
||||
- cephnfses
|
||||
- cephnvmeofgateways
|
||||
- cephobjectstores
|
||||
- cephobjectstoreusers
|
||||
- cephobjectrealms
|
||||
@@ -169,6 +170,7 @@ rules:
|
||||
- cephblockpools/status
|
||||
- cephfilesystems/status
|
||||
- cephnfses/status
|
||||
- cephnvmeofgateways/status
|
||||
- cephobjectstores/status
|
||||
- cephobjectstoreusers/status
|
||||
- cephobjectrealms/status
|
||||
@@ -192,6 +194,7 @@ rules:
|
||||
- cephblockpools/finalizers
|
||||
- cephfilesystems/finalizers
|
||||
- cephnfses/finalizers
|
||||
- cephnvmeofgateways/finalizers
|
||||
- cephobjectstores/finalizers
|
||||
- cephobjectstoreusers/finalizers
|
||||
- cephobjectrealms/finalizers
|
||||
|
||||
@@ -11510,6 +11510,920 @@ spec:
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.19.0
|
||||
helm.sh/resource-policy: keep
|
||||
name: cephnvmeofgateways.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephNVMeOFGateway
|
||||
listKind: CephNVMeOFGatewayList
|
||||
plural: cephnvmeofgateways
|
||||
shortNames:
|
||||
- nvmeof
|
||||
singular: cephnvmeofgateway
|
||||
scope: Namespaced
|
||||
versions:
|
||||
- additionalPrinterColumns:
|
||||
- jsonPath: .status.phase
|
||||
name: Phase
|
||||
type: string
|
||||
- jsonPath: .metadata.creationTimestamp
|
||||
name: Age
|
||||
type: date
|
||||
name: v1
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
|
||||
properties:
|
||||
apiVersion:
|
||||
description: |-
|
||||
APIVersion defines the versioned schema of this representation of an object.
|
||||
Servers should convert recognized schemas to the latest internal value, and
|
||||
may reject unrecognized values.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||
type: string
|
||||
kind:
|
||||
description: |-
|
||||
Kind is a string value representing the REST resource this object represents.
|
||||
Servers may infer this from the endpoint the client submits requests to.
|
||||
Cannot be updated.
|
||||
In CamelCase.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||
type: string
|
||||
metadata:
|
||||
type: object
|
||||
spec:
|
||||
description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
|
||||
properties:
|
||||
annotations:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: The annotations-related configuration to add/set on each Pod related object.
|
||||
type: object
|
||||
configMapRef:
|
||||
description: |-
|
||||
ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
|
||||
If not specified, a default configuration will be generated
|
||||
minLength: 1
|
||||
type: string
|
||||
group:
|
||||
description: Group is the gateway group name for high availability (ANA group)
|
||||
minLength: 1
|
||||
type: string
|
||||
hostNetwork:
|
||||
description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
|
||||
type: boolean
|
||||
image:
|
||||
description: |-
|
||||
Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
minLength: 1
|
||||
type: string
|
||||
instances:
|
||||
description: The number of active gateway instances
|
||||
minimum: 1
|
||||
type: integer
|
||||
labels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: The labels-related configuration to add/set on each Pod related object.
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: |-
|
||||
A liveness-probe to verify that gateway has valid run-time state.
|
||||
If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
|
||||
properties:
|
||||
disabled:
|
||||
description: Disabled determines whether probe is disable or not
|
||||
type: boolean
|
||||
probe:
|
||||
description: |-
|
||||
Probe describes a health check to be performed against a container to determine whether it is
|
||||
alive or ready to receive traffic.
|
||||
properties:
|
||||
exec:
|
||||
description: Exec specifies a command to execute in the container.
|
||||
properties:
|
||||
command:
|
||||
description: |-
|
||||
Command is the command line to execute inside the container, the working directory for the
|
||||
command is root ('/') in the container's filesystem. The command is simply exec'd, it is
|
||||
not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
|
||||
a shell, you need to explicitly call out to that shell.
|
||||
Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
failureThreshold:
|
||||
description: |-
|
||||
Minimum consecutive failures for the probe to be considered failed after having succeeded.
|
||||
Defaults to 3. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
grpc:
|
||||
description: GRPC specifies a GRPC HealthCheckRequest.
|
||||
properties:
|
||||
port:
|
||||
description: Port number of the gRPC service. Number must be in the range 1 to 65535.
|
||||
format: int32
|
||||
type: integer
|
||||
service:
|
||||
default: ""
|
||||
description: |-
|
||||
Service is the name of the service to place in the gRPC HealthCheckRequest
|
||||
(see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
|
||||
|
||||
If this is not specified, the default behavior is defined by gRPC.
|
||||
type: string
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
httpGet:
|
||||
description: HTTPGet specifies an HTTP GET request to perform.
|
||||
properties:
|
||||
host:
|
||||
description: |-
|
||||
Host name to connect to, defaults to the pod IP. You probably want to set
|
||||
"Host" in httpHeaders instead.
|
||||
type: string
|
||||
httpHeaders:
|
||||
description: Custom headers to set in the request. HTTP allows repeated headers.
|
||||
items:
|
||||
description: HTTPHeader describes a custom header to be used in HTTP probes
|
||||
properties:
|
||||
name:
|
||||
description: |-
|
||||
The header field name.
|
||||
This will be canonicalized upon output, so case-variant names will be understood as the same header.
|
||||
type: string
|
||||
value:
|
||||
description: The header field value
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- value
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
path:
|
||||
description: Path to access on the HTTP server.
|
||||
type: string
|
||||
port:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
description: |-
|
||||
Name or number of the port to access on the container.
|
||||
Number must be in the range 1 to 65535.
|
||||
Name must be an IANA_SVC_NAME.
|
||||
x-kubernetes-int-or-string: true
|
||||
scheme:
|
||||
description: |-
|
||||
Scheme to use for connecting to the host.
|
||||
Defaults to HTTP.
|
||||
type: string
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
initialDelaySeconds:
|
||||
description: |-
|
||||
Number of seconds after the container has started before liveness probes are initiated.
|
||||
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
format: int32
|
||||
type: integer
|
||||
periodSeconds:
|
||||
description: |-
|
||||
How often (in seconds) to perform the probe.
|
||||
Default to 10 seconds. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
successThreshold:
|
||||
description: |-
|
||||
Minimum consecutive successes for the probe to be considered successful after having failed.
|
||||
Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
tcpSocket:
|
||||
description: TCPSocket specifies a connection to a TCP port.
|
||||
properties:
|
||||
host:
|
||||
description: 'Optional: Host name to connect to, defaults to the pod IP.'
|
||||
type: string
|
||||
port:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
description: |-
|
||||
Number or name of the port to access on the container.
|
||||
Number must be in the range 1 to 65535.
|
||||
Name must be an IANA_SVC_NAME.
|
||||
x-kubernetes-int-or-string: true
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
terminationGracePeriodSeconds:
|
||||
format: int64
|
||||
type: integer
|
||||
timeoutSeconds:
|
||||
description: |-
|
||||
Number of seconds after which the probe times out.
|
||||
Defaults to 1 second. Minimum value is 1.
|
||||
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
nvmeofConfig:
|
||||
additionalProperties:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
description: |-
|
||||
NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
|
||||
This allows users to override or add configuration options without needing to manage a ConfigMap
|
||||
type: object
|
||||
placement:
|
||||
properties:
|
||||
nodeAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
preference:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchFields:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- preference
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
properties:
|
||||
nodeSelectorTerms:
|
||||
items:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchFields:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- nodeSelectorTerms
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
type: object
|
||||
podAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
podAffinityTerm:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- podAffinityTerm
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
podAntiAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
podAffinityTerm:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- podAffinityTerm
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
tolerations:
|
||||
items:
|
||||
properties:
|
||||
effect:
|
||||
type: string
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
tolerationSeconds:
|
||||
format: int64
|
||||
type: integer
|
||||
value:
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
topologySpreadConstraints:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
maxSkew:
|
||||
format: int32
|
||||
type: integer
|
||||
minDomains:
|
||||
format: int32
|
||||
type: integer
|
||||
nodeAffinityPolicy:
|
||||
type: string
|
||||
nodeTaintsPolicy:
|
||||
type: string
|
||||
topologyKey:
|
||||
type: string
|
||||
whenUnsatisfiable:
|
||||
type: string
|
||||
required:
|
||||
- maxSkew
|
||||
- topologyKey
|
||||
- whenUnsatisfiable
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
pool:
|
||||
description: Pool is the RADOS pool where NVMe-oF configuration is stored
|
||||
minLength: 1
|
||||
type: string
|
||||
ports:
|
||||
description: Ports configuration for the NVMe-oF gateway
|
||||
properties:
|
||||
discoveryPort:
|
||||
description: 'DiscoveryPort is the port for discovery service (default: 8009)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
gatewayPort:
|
||||
description: 'GatewayPort is the port for the gateway service (default: 5500)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
ioPort:
|
||||
description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
monitorPort:
|
||||
description: 'MonitorPort is the port for the monitor service (default: 5499)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
type: object
|
||||
priorityClassName:
|
||||
description: PriorityClassName sets the priority class on the pods
|
||||
type: string
|
||||
resources:
|
||||
description: Resources set resource requests and limits
|
||||
properties:
|
||||
claims:
|
||||
description: |-
|
||||
Claims lists the names of resources, defined in spec.resourceClaims,
|
||||
that are used by this container.
|
||||
|
||||
This field depends on the
|
||||
DynamicResourceAllocation feature gate.
|
||||
|
||||
This field is immutable. It can only be set for containers.
|
||||
items:
|
||||
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
|
||||
properties:
|
||||
name:
|
||||
description: |-
|
||||
Name must match the name of one entry in pod.spec.resourceClaims of
|
||||
the Pod where this field is used. It makes that resource available
|
||||
inside a container.
|
||||
type: string
|
||||
request:
|
||||
description: |-
|
||||
Request is the name chosen for a request in the referenced claim.
|
||||
If empty, everything from the claim is made available, otherwise
|
||||
only the result of this request.
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-map-keys:
|
||||
- name
|
||||
x-kubernetes-list-type: map
|
||||
limits:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Limits describes the maximum amount of compute resources allowed.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
requests:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Requests describes the minimum amount of compute resources required.
|
||||
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
|
||||
otherwise to an implementation-defined value. Requests cannot exceed Limits.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
required:
|
||||
- group
|
||||
- image
|
||||
- instances
|
||||
- pool
|
||||
type: object
|
||||
status:
|
||||
description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
|
||||
properties:
|
||||
cephx:
|
||||
properties:
|
||||
daemon:
|
||||
description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
|
||||
properties:
|
||||
keyCephVersion:
|
||||
description: |-
|
||||
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
|
||||
same string format as reported by `CephCluster.status.version.version` to allow them to be
|
||||
compared. E.g., `20.2.0-0`.
|
||||
For all newly-created resources, this field set to the version of Ceph that created the key.
|
||||
The special value "Uninitialized" indicates that keys are being created for the first time.
|
||||
An empty string indicates that the version is unknown, as expected in brownfield deployments.
|
||||
type: string
|
||||
keyGeneration:
|
||||
description: |-
|
||||
KeyGeneration represents the CephX key generation for the last successful reconcile.
|
||||
For all newly-created resources, this field is set to `1`.
|
||||
When keys are rotated due to any rotation policy, the generation is incremented or updated to
|
||||
the configured policy generation.
|
||||
Generation `0` indicates that keys existed prior to the implementation of key tracking.
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
conditions:
|
||||
items:
|
||||
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
|
||||
properties:
|
||||
lastHeartbeatTime:
|
||||
format: date-time
|
||||
type: string
|
||||
lastTransitionTime:
|
||||
format: date-time
|
||||
type: string
|
||||
message:
|
||||
type: string
|
||||
reason:
|
||||
description: ConditionReason is a reason for a condition
|
||||
type: string
|
||||
status:
|
||||
type: string
|
||||
type:
|
||||
description: ConditionType represent a resource's status
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
observedGeneration:
|
||||
description: ObservedGeneration is the latest generation observed by the controller.
|
||||
format: int64
|
||||
type: integer
|
||||
phase:
|
||||
type: string
|
||||
type: object
|
||||
required:
|
||||
- metadata
|
||||
- spec
|
||||
type: object
|
||||
served: true
|
||||
storage: true
|
||||
subresources:
|
||||
status: {}
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.19.0
|
||||
|
||||
@@ -364,6 +364,7 @@ rules:
|
||||
- cephblockpools
|
||||
- cephfilesystems
|
||||
- cephnfses
|
||||
- cephnvmeofgateways
|
||||
- cephobjectstores
|
||||
- cephobjectstoreusers
|
||||
- cephobjectrealms
|
||||
@@ -390,6 +391,7 @@ rules:
|
||||
- cephblockpools/status
|
||||
- cephfilesystems/status
|
||||
- cephnfses/status
|
||||
- cephnvmeofgateways/status
|
||||
- cephobjectstores/status
|
||||
- cephobjectstoreusers/status
|
||||
- cephobjectrealms/status
|
||||
@@ -413,6 +415,7 @@ rules:
|
||||
- cephblockpools/finalizers
|
||||
- cephfilesystems/finalizers
|
||||
- cephnfses/finalizers
|
||||
- cephnvmeofgateways/finalizers
|
||||
- cephobjectstores/finalizers
|
||||
- cephobjectstoreusers/finalizers
|
||||
- cephobjectrealms/finalizers
|
||||
@@ -1002,6 +1005,7 @@ rules:
|
||||
- cephblockpools
|
||||
- cephfilesystems
|
||||
- cephnfses
|
||||
- cephnvmeofgateways
|
||||
- cephobjectstores
|
||||
- cephobjectstoreusers
|
||||
- cephobjectrealms
|
||||
@@ -1400,6 +1404,20 @@ metadata:
|
||||
# imagePullSecrets:
|
||||
# - name: my-registry-secret
|
||||
---
|
||||
# Service account for NVMe-oF gateway
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: rook-ceph-nvmeof
|
||||
namespace: rook-ceph # namespace:cluster
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
app.kubernetes.io/name: rook-ceph
|
||||
app.kubernetes.io/part-of: rook-ceph-operator
|
||||
# imagePullSecrets:
|
||||
# - name: my-registry-secret
|
||||
---
|
||||
# Service account for Ceph OSDs
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
|
||||
@@ -11503,6 +11503,919 @@ spec:
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.19.0
|
||||
name: cephnvmeofgateways.ceph.rook.io
|
||||
spec:
|
||||
group: ceph.rook.io
|
||||
names:
|
||||
kind: CephNVMeOFGateway
|
||||
listKind: CephNVMeOFGatewayList
|
||||
plural: cephnvmeofgateways
|
||||
shortNames:
|
||||
- nvmeof
|
||||
singular: cephnvmeofgateway
|
||||
scope: Namespaced
|
||||
versions:
|
||||
- additionalPrinterColumns:
|
||||
- jsonPath: .status.phase
|
||||
name: Phase
|
||||
type: string
|
||||
- jsonPath: .metadata.creationTimestamp
|
||||
name: Age
|
||||
type: date
|
||||
name: v1
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
description: CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
|
||||
properties:
|
||||
apiVersion:
|
||||
description: |-
|
||||
APIVersion defines the versioned schema of this representation of an object.
|
||||
Servers should convert recognized schemas to the latest internal value, and
|
||||
may reject unrecognized values.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
|
||||
type: string
|
||||
kind:
|
||||
description: |-
|
||||
Kind is a string value representing the REST resource this object represents.
|
||||
Servers may infer this from the endpoint the client submits requests to.
|
||||
Cannot be updated.
|
||||
In CamelCase.
|
||||
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
|
||||
type: string
|
||||
metadata:
|
||||
type: object
|
||||
spec:
|
||||
description: NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
|
||||
properties:
|
||||
annotations:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: The annotations-related configuration to add/set on each Pod related object.
|
||||
type: object
|
||||
configMapRef:
|
||||
description: |-
|
||||
ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
|
||||
If not specified, a default configuration will be generated
|
||||
minLength: 1
|
||||
type: string
|
||||
group:
|
||||
description: Group is the gateway group name for high availability (ANA group)
|
||||
minLength: 1
|
||||
type: string
|
||||
hostNetwork:
|
||||
description: Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
|
||||
type: boolean
|
||||
image:
|
||||
description: |-
|
||||
Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
For example, quay.io/ceph/nvmeof:1.5
|
||||
minLength: 1
|
||||
type: string
|
||||
instances:
|
||||
description: The number of active gateway instances
|
||||
minimum: 1
|
||||
type: integer
|
||||
labels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: The labels-related configuration to add/set on each Pod related object.
|
||||
type: object
|
||||
livenessProbe:
|
||||
description: |-
|
||||
A liveness-probe to verify that gateway has valid run-time state.
|
||||
If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
|
||||
properties:
|
||||
disabled:
|
||||
description: Disabled determines whether probe is disable or not
|
||||
type: boolean
|
||||
probe:
|
||||
description: |-
|
||||
Probe describes a health check to be performed against a container to determine whether it is
|
||||
alive or ready to receive traffic.
|
||||
properties:
|
||||
exec:
|
||||
description: Exec specifies a command to execute in the container.
|
||||
properties:
|
||||
command:
|
||||
description: |-
|
||||
Command is the command line to execute inside the container, the working directory for the
|
||||
command is root ('/') in the container's filesystem. The command is simply exec'd, it is
|
||||
not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
|
||||
a shell, you need to explicitly call out to that shell.
|
||||
Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
failureThreshold:
|
||||
description: |-
|
||||
Minimum consecutive failures for the probe to be considered failed after having succeeded.
|
||||
Defaults to 3. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
grpc:
|
||||
description: GRPC specifies a GRPC HealthCheckRequest.
|
||||
properties:
|
||||
port:
|
||||
description: Port number of the gRPC service. Number must be in the range 1 to 65535.
|
||||
format: int32
|
||||
type: integer
|
||||
service:
|
||||
default: ""
|
||||
description: |-
|
||||
Service is the name of the service to place in the gRPC HealthCheckRequest
|
||||
(see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
|
||||
|
||||
If this is not specified, the default behavior is defined by gRPC.
|
||||
type: string
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
httpGet:
|
||||
description: HTTPGet specifies an HTTP GET request to perform.
|
||||
properties:
|
||||
host:
|
||||
description: |-
|
||||
Host name to connect to, defaults to the pod IP. You probably want to set
|
||||
"Host" in httpHeaders instead.
|
||||
type: string
|
||||
httpHeaders:
|
||||
description: Custom headers to set in the request. HTTP allows repeated headers.
|
||||
items:
|
||||
description: HTTPHeader describes a custom header to be used in HTTP probes
|
||||
properties:
|
||||
name:
|
||||
description: |-
|
||||
The header field name.
|
||||
This will be canonicalized upon output, so case-variant names will be understood as the same header.
|
||||
type: string
|
||||
value:
|
||||
description: The header field value
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- value
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
path:
|
||||
description: Path to access on the HTTP server.
|
||||
type: string
|
||||
port:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
description: |-
|
||||
Name or number of the port to access on the container.
|
||||
Number must be in the range 1 to 65535.
|
||||
Name must be an IANA_SVC_NAME.
|
||||
x-kubernetes-int-or-string: true
|
||||
scheme:
|
||||
description: |-
|
||||
Scheme to use for connecting to the host.
|
||||
Defaults to HTTP.
|
||||
type: string
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
initialDelaySeconds:
|
||||
description: |-
|
||||
Number of seconds after the container has started before liveness probes are initiated.
|
||||
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
format: int32
|
||||
type: integer
|
||||
periodSeconds:
|
||||
description: |-
|
||||
How often (in seconds) to perform the probe.
|
||||
Default to 10 seconds. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
successThreshold:
|
||||
description: |-
|
||||
Minimum consecutive successes for the probe to be considered successful after having failed.
|
||||
Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
|
||||
format: int32
|
||||
type: integer
|
||||
tcpSocket:
|
||||
description: TCPSocket specifies a connection to a TCP port.
|
||||
properties:
|
||||
host:
|
||||
description: 'Optional: Host name to connect to, defaults to the pod IP.'
|
||||
type: string
|
||||
port:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
description: |-
|
||||
Number or name of the port to access on the container.
|
||||
Number must be in the range 1 to 65535.
|
||||
Name must be an IANA_SVC_NAME.
|
||||
x-kubernetes-int-or-string: true
|
||||
required:
|
||||
- port
|
||||
type: object
|
||||
terminationGracePeriodSeconds:
|
||||
format: int64
|
||||
type: integer
|
||||
timeoutSeconds:
|
||||
description: |-
|
||||
Number of seconds after which the probe times out.
|
||||
Defaults to 1 second. Minimum value is 1.
|
||||
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
nvmeofConfig:
|
||||
additionalProperties:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
description: |-
|
||||
NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
|
||||
This allows users to override or add configuration options without needing to manage a ConfigMap
|
||||
type: object
|
||||
placement:
|
||||
properties:
|
||||
nodeAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
preference:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchFields:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- preference
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
properties:
|
||||
nodeSelectorTerms:
|
||||
items:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchFields:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- nodeSelectorTerms
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
type: object
|
||||
podAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
podAffinityTerm:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- podAffinityTerm
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
podAntiAffinity:
|
||||
properties:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
podAffinityTerm:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
weight:
|
||||
format: int32
|
||||
type: integer
|
||||
required:
|
||||
- podAffinityTerm
|
||||
- weight
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
mismatchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
namespaceSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
namespaces:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
topologyKey:
|
||||
type: string
|
||||
required:
|
||||
- topologyKey
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
type: object
|
||||
tolerations:
|
||||
items:
|
||||
properties:
|
||||
effect:
|
||||
type: string
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
tolerationSeconds:
|
||||
format: int64
|
||||
type: integer
|
||||
value:
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
topologySpreadConstraints:
|
||||
items:
|
||||
properties:
|
||||
labelSelector:
|
||||
properties:
|
||||
matchExpressions:
|
||||
items:
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
operator:
|
||||
type: string
|
||||
values:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
required:
|
||||
- key
|
||||
- operator
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
matchLabels:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
type: object
|
||||
x-kubernetes-map-type: atomic
|
||||
matchLabelKeys:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
maxSkew:
|
||||
format: int32
|
||||
type: integer
|
||||
minDomains:
|
||||
format: int32
|
||||
type: integer
|
||||
nodeAffinityPolicy:
|
||||
type: string
|
||||
nodeTaintsPolicy:
|
||||
type: string
|
||||
topologyKey:
|
||||
type: string
|
||||
whenUnsatisfiable:
|
||||
type: string
|
||||
required:
|
||||
- maxSkew
|
||||
- topologyKey
|
||||
- whenUnsatisfiable
|
||||
type: object
|
||||
type: array
|
||||
type: object
|
||||
pool:
|
||||
description: Pool is the RADOS pool where NVMe-oF configuration is stored
|
||||
minLength: 1
|
||||
type: string
|
||||
ports:
|
||||
description: Ports configuration for the NVMe-oF gateway
|
||||
properties:
|
||||
discoveryPort:
|
||||
description: 'DiscoveryPort is the port for discovery service (default: 8009)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
gatewayPort:
|
||||
description: 'GatewayPort is the port for the gateway service (default: 5500)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
ioPort:
|
||||
description: 'IOPort is the port for NVMe-oF IO traffic (default: 4420)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
monitorPort:
|
||||
description: 'MonitorPort is the port for the monitor service (default: 5499)'
|
||||
format: int32
|
||||
maximum: 65535
|
||||
minimum: 1
|
||||
type: integer
|
||||
type: object
|
||||
priorityClassName:
|
||||
description: PriorityClassName sets the priority class on the pods
|
||||
type: string
|
||||
resources:
|
||||
description: Resources set resource requests and limits
|
||||
properties:
|
||||
claims:
|
||||
description: |-
|
||||
Claims lists the names of resources, defined in spec.resourceClaims,
|
||||
that are used by this container.
|
||||
|
||||
This field depends on the
|
||||
DynamicResourceAllocation feature gate.
|
||||
|
||||
This field is immutable. It can only be set for containers.
|
||||
items:
|
||||
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
|
||||
properties:
|
||||
name:
|
||||
description: |-
|
||||
Name must match the name of one entry in pod.spec.resourceClaims of
|
||||
the Pod where this field is used. It makes that resource available
|
||||
inside a container.
|
||||
type: string
|
||||
request:
|
||||
description: |-
|
||||
Request is the name chosen for a request in the referenced claim.
|
||||
If empty, everything from the claim is made available, otherwise
|
||||
only the result of this request.
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
type: object
|
||||
type: array
|
||||
x-kubernetes-list-map-keys:
|
||||
- name
|
||||
x-kubernetes-list-type: map
|
||||
limits:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Limits describes the maximum amount of compute resources allowed.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
requests:
|
||||
additionalProperties:
|
||||
anyOf:
|
||||
- type: integer
|
||||
- type: string
|
||||
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||
x-kubernetes-int-or-string: true
|
||||
description: |-
|
||||
Requests describes the minimum amount of compute resources required.
|
||||
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
|
||||
otherwise to an implementation-defined value. Requests cannot exceed Limits.
|
||||
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
||||
type: object
|
||||
type: object
|
||||
required:
|
||||
- group
|
||||
- image
|
||||
- instances
|
||||
- pool
|
||||
type: object
|
||||
status:
|
||||
description: NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
|
||||
properties:
|
||||
cephx:
|
||||
properties:
|
||||
daemon:
|
||||
description: Daemon shows the CephX key status for local Ceph daemons associated with this resources.
|
||||
properties:
|
||||
keyCephVersion:
|
||||
description: |-
|
||||
KeyCephVersion reports the Ceph version that created the current generation's keys. This is
|
||||
same string format as reported by `CephCluster.status.version.version` to allow them to be
|
||||
compared. E.g., `20.2.0-0`.
|
||||
For all newly-created resources, this field set to the version of Ceph that created the key.
|
||||
The special value "Uninitialized" indicates that keys are being created for the first time.
|
||||
An empty string indicates that the version is unknown, as expected in brownfield deployments.
|
||||
type: string
|
||||
keyGeneration:
|
||||
description: |-
|
||||
KeyGeneration represents the CephX key generation for the last successful reconcile.
|
||||
For all newly-created resources, this field is set to `1`.
|
||||
When keys are rotated due to any rotation policy, the generation is incremented or updated to
|
||||
the configured policy generation.
|
||||
Generation `0` indicates that keys existed prior to the implementation of key tracking.
|
||||
format: int32
|
||||
type: integer
|
||||
type: object
|
||||
type: object
|
||||
conditions:
|
||||
items:
|
||||
description: Condition represents a status condition on any Rook-Ceph Custom Resource.
|
||||
properties:
|
||||
lastHeartbeatTime:
|
||||
format: date-time
|
||||
type: string
|
||||
lastTransitionTime:
|
||||
format: date-time
|
||||
type: string
|
||||
message:
|
||||
type: string
|
||||
reason:
|
||||
description: ConditionReason is a reason for a condition
|
||||
type: string
|
||||
status:
|
||||
type: string
|
||||
type:
|
||||
description: ConditionType represent a resource's status
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
observedGeneration:
|
||||
description: ObservedGeneration is the latest generation observed by the controller.
|
||||
format: int64
|
||||
type: integer
|
||||
phase:
|
||||
type: string
|
||||
type: object
|
||||
required:
|
||||
- metadata
|
||||
- spec
|
||||
type: object
|
||||
served: true
|
||||
storage: true
|
||||
subresources:
|
||||
status: {}
|
||||
---
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
annotations:
|
||||
controller-gen.kubebuilder.io/version: v0.19.0
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: nvmeof.csi.ceph.com-nodeplugin
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app: nvmeof.csi.ceph.com-nodeplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: nvmeof.csi.ceph.com-nodeplugin
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- --v=5
|
||||
- --type=nvmeof
|
||||
- --nodeserver=true
|
||||
- --nodeid=$(NODE_ID)
|
||||
- --drivername=nvmeof.csi.ceph.com
|
||||
- --endpoint=unix:///csi/csi.sock
|
||||
- --pidlimit=-1
|
||||
- --csi-addons-endpoint=unix:///csi/csi-addons.sock
|
||||
- --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/
|
||||
- --logtostderr=false
|
||||
- --alsologtostderr=true
|
||||
- --log_file=csi-logs/csi-nvmeofplugin.log
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.podIP
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
image: quay.io/cephcsi/cephcsi:v3.16.0
|
||||
imagePullPolicy: Always
|
||||
name: csi-nvmeofplugin
|
||||
resources: {}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
add:
|
||||
- SYS_ADMIN
|
||||
drop:
|
||||
- All
|
||||
privileged: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /run/mount
|
||||
name: host-run-mount
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
- mountPath: /tmp/csi/keys
|
||||
name: keys-tmp-dir
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
# FIXED: Mount path should end with / to mount as directory
|
||||
- mountPath: /etc/ceph-csi-config/
|
||||
name: ceph-csi-config
|
||||
- mountPath: /var/lib/kubelet/plugins
|
||||
mountPropagation: Bidirectional
|
||||
name: plugin-mount-dir
|
||||
- mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: Bidirectional
|
||||
name: pods-mount-dir
|
||||
- mountPath: /etc/selinux
|
||||
name: etc-selinux
|
||||
readOnly: true
|
||||
- mountPath: /run/secrets/tokens
|
||||
name: oidc-token
|
||||
readOnly: true
|
||||
- mountPath: /csi-logs
|
||||
name: logs-dir
|
||||
# Log rotation mount - commented out since ConfigMap is optional
|
||||
# - mountPath: /etc/logrotate.d
|
||||
# name: log-rotate-dir
|
||||
# readOnly: true
|
||||
- args:
|
||||
- "--v=1"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock"
|
||||
image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: driver-registrar
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- All
|
||||
privileged: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
- mountPath: /registration
|
||||
name: registration-dir
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
priorityClassName: system-node-critical
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
# FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend
|
||||
# This service account already exists and has the necessary permissions
|
||||
serviceAccount: rook-csi-rbd-plugin-sa
|
||||
serviceAccountName: rook-csi-rbd-plugin-sa
|
||||
terminationGracePeriodSeconds: 30
|
||||
# tolerations:
|
||||
# - effect: NoSchedule
|
||||
# key: node.ocs.openshift.io/storage
|
||||
# operator: Equal
|
||||
# value: "true"
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /dev
|
||||
type: ""
|
||||
name: host-dev
|
||||
- hostPath:
|
||||
path: /sys
|
||||
type: ""
|
||||
name: host-sys
|
||||
- hostPath:
|
||||
path: /run/mount
|
||||
type: ""
|
||||
name: host-run-mount
|
||||
- hostPath:
|
||||
path: /lib/modules
|
||||
type: ""
|
||||
name: lib-modules
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
name: keys-tmp-dir
|
||||
# FIXED: Use rook-ceph-csi-config instead of ceph-csi-config
|
||||
# This ConfigMap contains the cluster configuration with monitor endpoints
|
||||
- configMap:
|
||||
name: rook-ceph-csi-config
|
||||
items:
|
||||
- key: csi-cluster-config-json
|
||||
path: config.json
|
||||
name: ceph-csi-config
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
name: plugin-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins
|
||||
type: Directory
|
||||
name: plugin-mount-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
name: pods-mount-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry
|
||||
type: Directory
|
||||
name: registration-dir
|
||||
- hostPath:
|
||||
path: /etc/selinux
|
||||
type: ""
|
||||
name: etc-selinux
|
||||
- name: oidc-token
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: ceph-csi-kms
|
||||
expirationSeconds: 3600
|
||||
path: oidc-token
|
||||
- hostPath:
|
||||
path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin
|
||||
type: DirectoryOrCreate
|
||||
name: logs-dir
|
||||
# Log rotation ConfigMap - optional, can be created if log rotation is needed
|
||||
# - configMap:
|
||||
# defaultMode: 420
|
||||
# name: nvmeof.csi.ceph.com-logrotate-config
|
||||
# name: log-rotate-dir
|
||||
updateStrategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
type: RollingUpdate
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephBlockPool
|
||||
metadata:
|
||||
name: nvmeof
|
||||
namespace: rook-ceph # namespace:cluster
|
||||
spec:
|
||||
failureDomain: host
|
||||
replicated:
|
||||
size: 3
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: nvmeof-test-pod
|
||||
namespace: default
|
||||
spec:
|
||||
containers:
|
||||
- name: test-container
|
||||
image: busybox
|
||||
command: ["/bin/sh"]
|
||||
args:
|
||||
[
|
||||
"-c",
|
||||
"while true; do echo 'Testing NVMe-oF mount'; df -h /mnt/nvmeof; sleep 45; done",
|
||||
]
|
||||
volumeMounts:
|
||||
- name: nvmeof-volume
|
||||
mountPath: /mnt/nvmeof
|
||||
volumes:
|
||||
- name: nvmeof-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: nvmeof-external-volume
|
||||
restartPolicy: Never
|
||||
@@ -0,0 +1,336 @@
|
||||
---
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update", "patch", "create"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots/status"]
|
||||
verbs: ["get", "list", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments/status"]
|
||||
verbs: ["patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["csinodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["serviceaccounts"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["serviceaccounts/token"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotcontents"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: ["replication.storage.openshift.io"]
|
||||
resources: ["volumegroupreplicationcontents"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["replication.storage.openshift.io"]
|
||||
resources: ["volumegroupreplicationclasses"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattributesclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims/status"]
|
||||
verbs: ["patch"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: nvmeof-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
name: nvmeof-external-provisioner-cfg
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "delete"]
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner-role-cfg
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: nvmeof-external-provisioner-cfg
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
# ---
|
||||
# # ssc based on the Ceph-CSI ssc
|
||||
# # the "users:" contains a reference to the "rook-ceph" namespace
|
||||
# #
|
||||
# kind: SecurityContextConstraints
|
||||
# apiVersion: security.openshift.io/v1
|
||||
# metadata:
|
||||
# name: "ceph-nvmeof-provisioner"
|
||||
# # To allow running privilegedContainers
|
||||
# allowPrivilegedContainer: true
|
||||
# # SYS_ADMIN is needed for rbd to execute rbd map command
|
||||
# allowedCapabilities:
|
||||
# - "SYS_ADMIN"
|
||||
# # Set to false as we write to RootFilesystem inside container
|
||||
# readOnlyRootFilesystem: false
|
||||
# requiredDropCapabilities:
|
||||
# - ALL
|
||||
# # The type of volumes which are mounted to csi pods
|
||||
# volumes:
|
||||
# - configMap
|
||||
# - projected
|
||||
# - emptyDir
|
||||
# runAsUser:
|
||||
# type: RunAsAny
|
||||
# seLinuxContext:
|
||||
# type: RunAsAny
|
||||
# users:
|
||||
# # A user needs to be added for each service account.
|
||||
# - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner"
|
||||
---
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: csi-nvmeofplugin-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-nvmeofplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
openshift.io/scc: rook-ceph-csi
|
||||
labels:
|
||||
app: csi-nvmeofplugin-provisioner
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- csi-nvmeofplugin-provisioner
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
serviceAccountName: nvmeof-csi-provisioner
|
||||
priorityClassName: system-cluster-critical
|
||||
containers:
|
||||
- name: csi-nvmeofplugin
|
||||
# for stable functionality replace canary with latest release version
|
||||
# image: quay.io/gdidi/cephcsi:nvmeof
|
||||
image: quay.io/cephcsi/cephcsi:v3.16.0
|
||||
command: ["/usr/local/bin/cephcsi"]
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--type=nvmeof"
|
||||
- "--controllerserver=true"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=nvmeof.csi.ceph.com"
|
||||
- "--pidlimit=-1"
|
||||
- "--rbdhardmaxclonedepth=8"
|
||||
- "--rbdsoftmaxclonedepth=4"
|
||||
- "--enableprofiling=false"
|
||||
- "--setmetadata=true"
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: ceph-csi-config
|
||||
mountPath: /etc/ceph-csi-config/
|
||||
- name: keys-tmp-dir
|
||||
mountPath: /tmp/csi/keys
|
||||
- name: oidc-token
|
||||
mountPath: /run/secrets/tokens
|
||||
readOnly: true
|
||||
- name: csi-provisioner
|
||||
# image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0
|
||||
image: quay.io/nixpanic/csi-provisioner:pr1440
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=1"
|
||||
- "--timeout=150s"
|
||||
- "--retry-interval-start=500ms"
|
||||
- "--leader-election=true"
|
||||
- "--feature-gates=HonorPVReclaimPolicy=true"
|
||||
- "--prevent-volume-mode-conversion=true"
|
||||
# if fstype is not specified in storageclass, ext4 is default
|
||||
- "--default-fstype=ext4"
|
||||
- "--extra-create-metadata=true"
|
||||
- "--immediate-topology=false"
|
||||
- "--http-endpoint=$(POD_IP):8090"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
ports:
|
||||
- containerPort: 8090
|
||||
name: provisioner
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: csi-attacher
|
||||
image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0
|
||||
args:
|
||||
- "--v=1"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--leader-election=true"
|
||||
- "--retry-interval-start=500ms"
|
||||
- "--default-fstype=ext4"
|
||||
- "--http-endpoint=$(POD_IP):8093"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi-provisioner.sock
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
ports:
|
||||
- containerPort: 8093
|
||||
name: attacher
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: csi-resizer
|
||||
image: quay.io/nixpanic/csi-resizer:pr544
|
||||
# image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
- "--leader-election=true"
|
||||
- "--handle-volume-inuse-error=false"
|
||||
- "--feature-gates=VolumeAttributesClass=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
emptyDir:
|
||||
medium: "Memory"
|
||||
# FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config
|
||||
# This ConfigMap contains the cluster configuration with monitor endpoints
|
||||
# that Ceph-CSI needs to connect to the Ceph cluster
|
||||
- name: ceph-csi-config
|
||||
configMap:
|
||||
name: rook-ceph-csi-config
|
||||
items:
|
||||
- key: csi-cluster-config-json
|
||||
path: config.json
|
||||
- name: keys-tmp-dir
|
||||
emptyDir:
|
||||
medium: "Memory"
|
||||
# Only if nvmeof support encryption
|
||||
- name: oidc-token
|
||||
projected:
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
path: oidc-token
|
||||
expirationSeconds: 3600
|
||||
audience: ceph-csi-kms
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: nvmeof-external-volume
|
||||
namespace: default
|
||||
spec:
|
||||
storageClassName: ceph-nvmeof
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 128Mi
|
||||
@@ -0,0 +1,31 @@
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: ceph-nvmeof
|
||||
parameters:
|
||||
clusterID: rook-ceph
|
||||
pool: nvmeof
|
||||
subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
|
||||
# Management API - talks to gateway to create subsystems/namespaces
|
||||
nvmeofGatewayAddress: "172.30.47.120"
|
||||
nvmeofGatewayPort: "5500"
|
||||
# Data Plane - worker nodes connect here for actual I/O
|
||||
# List ALL gateway pods for HA and multipath
|
||||
listeners: |
|
||||
[
|
||||
{
|
||||
"address": "10.129.2.36",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-a"
|
||||
}
|
||||
]
|
||||
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
|
||||
provisioner: nvmeof.csi.ceph.com
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
allowVolumeExpansion: false
|
||||
@@ -0,0 +1,13 @@
|
||||
# This example is for Ceph v20 and above only
|
||||
|
||||
apiVersion: ceph.rook.io/v1
|
||||
kind: CephNVMeOFGateway
|
||||
metadata:
|
||||
name: nvmeof
|
||||
namespace: rook-ceph # namespace:cluster
|
||||
spec:
|
||||
image: quay.io/ceph/nvmeof:1.5
|
||||
pool: nvmeof
|
||||
group: group-a
|
||||
instances: 1
|
||||
hostNetwork: false
|
||||
@@ -18,7 +18,7 @@ allowHostNetwork: false
|
||||
# set to true if running rook with the provider as host
|
||||
allowHostPorts: false
|
||||
priority:
|
||||
allowedCapabilities: ["MKNOD"]
|
||||
allowedCapabilities: ["MKNOD", "SYS_ADMIN", "NET_RAW"]
|
||||
allowHostIPC: true
|
||||
readOnlyRootFilesystem: false
|
||||
# drop all default privileges
|
||||
@@ -49,6 +49,7 @@ users:
|
||||
- system:serviceaccount:rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster
|
||||
- system:serviceaccount:rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster
|
||||
- system:serviceaccount:rook-ceph:rook-ceph-rgw # serviceaccount:namespace:cluster
|
||||
- system:serviceaccount:rook-ceph:rook-ceph-nvmeof # serviceaccount:namespace:cluster
|
||||
---
|
||||
# scc for the CSI driver
|
||||
kind: SecurityContextConstraints
|
||||
@@ -97,6 +98,7 @@ users:
|
||||
- system:serviceaccount:rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:rook-csi-nfs-plugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:rook-csi-nfs-provisioner-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator
|
||||
# Service accounts for the csi operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-cephfs-ctrlplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
Copyright 2024 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1
|
||||
|
||||
// IsHostNetwork returns true if host networking is enabled for this gateway.
|
||||
// If not explicitly set on the gateway spec, it inherits from the cluster spec.
|
||||
func (n *CephNVMeOFGateway) IsHostNetwork(c *ClusterSpec) bool {
|
||||
if n.Spec.HostNetwork != nil {
|
||||
return *n.Spec.HostNetwork
|
||||
}
|
||||
return c.Network.IsHost()
|
||||
}
|
||||
@@ -65,6 +65,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
|
||||
&CephFilesystemList{},
|
||||
&CephNFS{},
|
||||
&CephNFSList{},
|
||||
&CephNVMeOFGateway{},
|
||||
&CephNVMeOFGatewayList{},
|
||||
&CephObjectStore{},
|
||||
&CephObjectStoreList{},
|
||||
&CephObjectStoreUser{},
|
||||
|
||||
@@ -3929,3 +3929,125 @@ const (
|
||||
// Always means the Ceph COSI driver will be deployed even if the object store is not present
|
||||
COSIDeploymentStrategyAlways COSIDeploymentStrategy = "Always"
|
||||
)
|
||||
|
||||
// +genclient
|
||||
// +genclient:noStatus
|
||||
// +kubebuilder:resource:shortName=nvmeof,path=cephnvmeofgateways
|
||||
// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase`
|
||||
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
|
||||
// +kubebuilder:subresource:status
|
||||
//
|
||||
// CephNVMeOFGateway represents a Ceph NVMe-oF Gateway
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type CephNVMeOFGateway struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata"`
|
||||
Spec NVMeOFGatewaySpec `json:"spec"`
|
||||
// +optional
|
||||
Status *NVMeOFGatewayStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
// NVMeOFGatewayStatus represents the status of Ceph NVMe-oF Gateway
|
||||
type NVMeOFGatewayStatus struct {
|
||||
Status `json:",inline"`
|
||||
Cephx LocalCephxStatus `json:"cephx,omitempty"`
|
||||
}
|
||||
|
||||
// CephNVMeOFGatewayList represents a list of Ceph NVMe-oF Gateways
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type CephNVMeOFGatewayList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata"`
|
||||
Items []CephNVMeOFGateway `json:"items"`
|
||||
}
|
||||
|
||||
// NVMeOFGatewaySpec represents the spec of an NVMe-oF gateway
|
||||
type NVMeOFGatewaySpec struct {
|
||||
// Image is the container image to use for the NVMe-oF gateway daemon.
|
||||
// For example, quay.io/ceph/nvmeof:1.5
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Image string `json:"image"`
|
||||
|
||||
// The number of active gateway instances
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
Instances int `json:"instances"`
|
||||
|
||||
// Pool is the RADOS pool where NVMe-oF configuration is stored
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Pool string `json:"pool"`
|
||||
|
||||
// Group is the gateway group name for high availability (ANA group)
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Group string `json:"group"`
|
||||
|
||||
// ConfigMapRef is the name of the ConfigMap containing nvmeof.conf configuration
|
||||
// If not specified, a default configuration will be generated
|
||||
// +optional
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
ConfigMapRef string `json:"configMapRef,omitempty"`
|
||||
|
||||
// NVMeOFConfig is a map of section names to key-value pairs for nvmeof.conf configuration
|
||||
// This allows users to override or add configuration options without needing to manage a ConfigMap
|
||||
// +optional
|
||||
NVMeOFConfig map[string]map[string]string `json:"nvmeofConfig,omitempty"`
|
||||
|
||||
// The affinity to place the gateway pods
|
||||
// +optional
|
||||
Placement Placement `json:"placement,omitempty"`
|
||||
|
||||
// The annotations-related configuration to add/set on each Pod related object.
|
||||
// +optional
|
||||
Annotations Annotations `json:"annotations,omitempty"`
|
||||
|
||||
// The labels-related configuration to add/set on each Pod related object.
|
||||
// +optional
|
||||
Labels Labels `json:"labels,omitempty"`
|
||||
|
||||
// Resources set resource requests and limits
|
||||
// +optional
|
||||
Resources v1.ResourceRequirements `json:"resources,omitempty"`
|
||||
|
||||
// PriorityClassName sets the priority class on the pods
|
||||
// +optional
|
||||
PriorityClassName string `json:"priorityClassName,omitempty"`
|
||||
|
||||
// Whether host networking is enabled for the gateway. If not set, the network settings from the cluster CR will be applied.
|
||||
// +optional
|
||||
HostNetwork *bool `json:"hostNetwork,omitempty"`
|
||||
|
||||
// Ports configuration for the NVMe-oF gateway
|
||||
// +optional
|
||||
Ports *NVMeOFGatewayPorts `json:"ports,omitempty"`
|
||||
|
||||
// A liveness-probe to verify that gateway has valid run-time state.
|
||||
// If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.
|
||||
// +optional
|
||||
LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"`
|
||||
}
|
||||
|
||||
// NVMeOFGatewayPorts represents the port configuration for NVMe-oF gateway
|
||||
type NVMeOFGatewayPorts struct {
|
||||
// IOPort is the port for NVMe-oF IO traffic (default: 4420)
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
// +kubebuilder:validation:Maximum=65535
|
||||
// +optional
|
||||
IOPort int32 `json:"ioPort,omitempty"`
|
||||
|
||||
// GatewayPort is the port for the gateway service (default: 5500)
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
// +kubebuilder:validation:Maximum=65535
|
||||
// +optional
|
||||
GatewayPort int32 `json:"gatewayPort,omitempty"`
|
||||
|
||||
// MonitorPort is the port for the monitor service (default: 5499)
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
// +kubebuilder:validation:Maximum=65535
|
||||
// +optional
|
||||
MonitorPort int32 `json:"monitorPort,omitempty"`
|
||||
|
||||
// DiscoveryPort is the port for discovery service (default: 8009)
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
// +kubebuilder:validation:Maximum=65535
|
||||
// +optional
|
||||
DiscoveryPort int32 `json:"discoveryPort,omitempty"`
|
||||
}
|
||||
@@ -1459,6 +1459,71 @@ func (in *CephNFSList) DeepCopyObject() runtime.Object {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *CephNVMeOFGateway) DeepCopyInto(out *CephNVMeOFGateway) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
|
||||
in.Spec.DeepCopyInto(&out.Spec)
|
||||
if in.Status != nil {
|
||||
in, out := &in.Status, &out.Status
|
||||
*out = new(NVMeOFGatewayStatus)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGateway.
|
||||
func (in *CephNVMeOFGateway) DeepCopy() *CephNVMeOFGateway {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(CephNVMeOFGateway)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *CephNVMeOFGateway) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *CephNVMeOFGatewayList) DeepCopyInto(out *CephNVMeOFGatewayList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]CephNVMeOFGateway, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephNVMeOFGatewayList.
|
||||
func (in *CephNVMeOFGatewayList) DeepCopy() *CephNVMeOFGatewayList {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(CephNVMeOFGatewayList)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *CephNVMeOFGatewayList) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *CephObjectRealm) DeepCopyInto(out *CephObjectRealm) {
|
||||
*out = *in
|
||||
@@ -3763,6 +3828,104 @@ func (in *NFSStatus) DeepCopy() *NFSStatus {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NVMeOFGatewayPorts) DeepCopyInto(out *NVMeOFGatewayPorts) {
|
||||
*out = *in
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayPorts.
|
||||
func (in *NVMeOFGatewayPorts) DeepCopy() *NVMeOFGatewayPorts {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(NVMeOFGatewayPorts)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NVMeOFGatewaySpec) DeepCopyInto(out *NVMeOFGatewaySpec) {
|
||||
*out = *in
|
||||
if in.NVMeOFConfig != nil {
|
||||
in, out := &in.NVMeOFConfig, &out.NVMeOFConfig
|
||||
*out = make(map[string]map[string]string, len(*in))
|
||||
for key, val := range *in {
|
||||
var outVal map[string]string
|
||||
if val == nil {
|
||||
(*out)[key] = nil
|
||||
} else {
|
||||
in, out := &val, &outVal
|
||||
*out = make(map[string]string, len(*in))
|
||||
for key, val := range *in {
|
||||
(*out)[key] = val
|
||||
}
|
||||
}
|
||||
(*out)[key] = outVal
|
||||
}
|
||||
}
|
||||
in.Placement.DeepCopyInto(&out.Placement)
|
||||
if in.Annotations != nil {
|
||||
in, out := &in.Annotations, &out.Annotations
|
||||
*out = make(Annotations, len(*in))
|
||||
for key, val := range *in {
|
||||
(*out)[key] = val
|
||||
}
|
||||
}
|
||||
if in.Labels != nil {
|
||||
in, out := &in.Labels, &out.Labels
|
||||
*out = make(Labels, len(*in))
|
||||
for key, val := range *in {
|
||||
(*out)[key] = val
|
||||
}
|
||||
}
|
||||
in.Resources.DeepCopyInto(&out.Resources)
|
||||
if in.HostNetwork != nil {
|
||||
in, out := &in.HostNetwork, &out.HostNetwork
|
||||
*out = new(bool)
|
||||
**out = **in
|
||||
}
|
||||
if in.Ports != nil {
|
||||
in, out := &in.Ports, &out.Ports
|
||||
*out = new(NVMeOFGatewayPorts)
|
||||
**out = **in
|
||||
}
|
||||
if in.LivenessProbe != nil {
|
||||
in, out := &in.LivenessProbe, &out.LivenessProbe
|
||||
*out = new(ProbeSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewaySpec.
|
||||
func (in *NVMeOFGatewaySpec) DeepCopy() *NVMeOFGatewaySpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(NVMeOFGatewaySpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NVMeOFGatewayStatus) DeepCopyInto(out *NVMeOFGatewayStatus) {
|
||||
*out = *in
|
||||
in.Status.DeepCopyInto(&out.Status)
|
||||
out.Cephx = in.Cephx
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NVMeOFGatewayStatus.
|
||||
func (in *NVMeOFGatewayStatus) DeepCopy() *NVMeOFGatewayStatus {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(NVMeOFGatewayStatus)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *NamedBlockPoolSpec) DeepCopyInto(out *NamedBlockPoolSpec) {
|
||||
*out = *in
|
||||
|
||||
@@ -39,6 +39,7 @@ type CephV1Interface interface {
|
||||
CephFilesystemMirrorsGetter
|
||||
CephFilesystemSubVolumeGroupsGetter
|
||||
CephNFSesGetter
|
||||
CephNVMeOFGatewaysGetter
|
||||
CephObjectRealmsGetter
|
||||
CephObjectStoresGetter
|
||||
CephObjectStoreUsersGetter
|
||||
@@ -96,6 +97,10 @@ func (c *CephV1Client) CephNFSes(namespace string) CephNFSInterface {
|
||||
return newCephNFSes(c, namespace)
|
||||
}
|
||||
|
||||
func (c *CephV1Client) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface {
|
||||
return newCephNVMeOFGateways(c, namespace)
|
||||
}
|
||||
|
||||
func (c *CephV1Client) CephObjectRealms(namespace string) CephObjectRealmInterface {
|
||||
return newCephObjectRealms(c, namespace)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by client-gen. DO NOT EDIT.
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
context "context"
|
||||
|
||||
cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
scheme "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
types "k8s.io/apimachinery/pkg/types"
|
||||
watch "k8s.io/apimachinery/pkg/watch"
|
||||
gentype "k8s.io/client-go/gentype"
|
||||
)
|
||||
|
||||
// CephNVMeOFGatewaysGetter has a method to return a CephNVMeOFGatewayInterface.
|
||||
// A group's client should implement this interface.
|
||||
type CephNVMeOFGatewaysGetter interface {
|
||||
CephNVMeOFGateways(namespace string) CephNVMeOFGatewayInterface
|
||||
}
|
||||
|
||||
// CephNVMeOFGatewayInterface has methods to work with CephNVMeOFGateway resources.
|
||||
type CephNVMeOFGatewayInterface interface {
|
||||
Create(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.CreateOptions) (*cephrookiov1.CephNVMeOFGateway, error)
|
||||
Update(ctx context.Context, cephNVMeOFGateway *cephrookiov1.CephNVMeOFGateway, opts metav1.UpdateOptions) (*cephrookiov1.CephNVMeOFGateway, error)
|
||||
Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
|
||||
DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
|
||||
Get(ctx context.Context, name string, opts metav1.GetOptions) (*cephrookiov1.CephNVMeOFGateway, error)
|
||||
List(ctx context.Context, opts metav1.ListOptions) (*cephrookiov1.CephNVMeOFGatewayList, error)
|
||||
Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
|
||||
Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *cephrookiov1.CephNVMeOFGateway, err error)
|
||||
CephNVMeOFGatewayExpansion
|
||||
}
|
||||
|
||||
// cephNVMeOFGateways implements CephNVMeOFGatewayInterface
|
||||
type cephNVMeOFGateways struct {
|
||||
*gentype.ClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList]
|
||||
}
|
||||
|
||||
// newCephNVMeOFGateways returns a CephNVMeOFGateways
|
||||
func newCephNVMeOFGateways(c *CephV1Client, namespace string) *cephNVMeOFGateways {
|
||||
return &cephNVMeOFGateways{
|
||||
gentype.NewClientWithList[*cephrookiov1.CephNVMeOFGateway, *cephrookiov1.CephNVMeOFGatewayList](
|
||||
"cephnvmeofgateways",
|
||||
c.RESTClient(),
|
||||
scheme.ParameterCodec,
|
||||
namespace,
|
||||
func() *cephrookiov1.CephNVMeOFGateway { return &cephrookiov1.CephNVMeOFGateway{} },
|
||||
func() *cephrookiov1.CephNVMeOFGatewayList { return &cephrookiov1.CephNVMeOFGatewayList{} },
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,10 @@ func (c *FakeCephV1) CephNFSes(namespace string) v1.CephNFSInterface {
|
||||
return newFakeCephNFSes(c, namespace)
|
||||
}
|
||||
|
||||
func (c *FakeCephV1) CephNVMeOFGateways(namespace string) v1.CephNVMeOFGatewayInterface {
|
||||
return newFakeCephNVMeOFGateways(c, namespace)
|
||||
}
|
||||
|
||||
func (c *FakeCephV1) CephObjectRealms(namespace string) v1.CephObjectRealmInterface {
|
||||
return newFakeCephObjectRealms(c, namespace)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by client-gen. DO NOT EDIT.
|
||||
|
||||
package fake
|
||||
|
||||
import (
|
||||
v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
cephrookiov1 "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1"
|
||||
gentype "k8s.io/client-go/gentype"
|
||||
)
|
||||
|
||||
// fakeCephNVMeOFGateways implements CephNVMeOFGatewayInterface
|
||||
type fakeCephNVMeOFGateways struct {
|
||||
*gentype.FakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList]
|
||||
Fake *FakeCephV1
|
||||
}
|
||||
|
||||
func newFakeCephNVMeOFGateways(fake *FakeCephV1, namespace string) cephrookiov1.CephNVMeOFGatewayInterface {
|
||||
return &fakeCephNVMeOFGateways{
|
||||
gentype.NewFakeClientWithList[*v1.CephNVMeOFGateway, *v1.CephNVMeOFGatewayList](
|
||||
fake.Fake,
|
||||
namespace,
|
||||
v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"),
|
||||
v1.SchemeGroupVersion.WithKind("CephNVMeOFGateway"),
|
||||
func() *v1.CephNVMeOFGateway { return &v1.CephNVMeOFGateway{} },
|
||||
func() *v1.CephNVMeOFGatewayList { return &v1.CephNVMeOFGatewayList{} },
|
||||
func(dst, src *v1.CephNVMeOFGatewayList) { dst.ListMeta = src.ListMeta },
|
||||
func(list *v1.CephNVMeOFGatewayList) []*v1.CephNVMeOFGateway {
|
||||
return gentype.ToPointerSlice(list.Items)
|
||||
},
|
||||
func(list *v1.CephNVMeOFGatewayList, items []*v1.CephNVMeOFGateway) {
|
||||
list.Items = gentype.FromPointerSlice(items)
|
||||
},
|
||||
),
|
||||
fake,
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,8 @@ type CephFilesystemSubVolumeGroupExpansion interface{}
|
||||
|
||||
type CephNFSExpansion interface{}
|
||||
|
||||
type CephNVMeOFGatewayExpansion interface{}
|
||||
|
||||
type CephObjectRealmExpansion interface{}
|
||||
|
||||
type CephObjectStoreExpansion interface{}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by informer-gen. DO NOT EDIT.
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
context "context"
|
||||
time "time"
|
||||
|
||||
apiscephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
versioned "github.com/rook/rook/pkg/client/clientset/versioned"
|
||||
internalinterfaces "github.com/rook/rook/pkg/client/informers/externalversions/internalinterfaces"
|
||||
cephrookiov1 "github.com/rook/rook/pkg/client/listers/ceph.rook.io/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
runtime "k8s.io/apimachinery/pkg/runtime"
|
||||
watch "k8s.io/apimachinery/pkg/watch"
|
||||
cache "k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
// CephNVMeOFGatewayInformer provides access to a shared informer and lister for
|
||||
// CephNVMeOFGateways.
|
||||
type CephNVMeOFGatewayInformer interface {
|
||||
Informer() cache.SharedIndexInformer
|
||||
Lister() cephrookiov1.CephNVMeOFGatewayLister
|
||||
}
|
||||
|
||||
type cephNVMeOFGatewayInformer struct {
|
||||
factory internalinterfaces.SharedInformerFactory
|
||||
tweakListOptions internalinterfaces.TweakListOptionsFunc
|
||||
namespace string
|
||||
}
|
||||
|
||||
// NewCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type.
|
||||
// Always prefer using an informer factory to get a shared informer instead of getting an independent
|
||||
// one. This reduces memory footprint and number of connections to the server.
|
||||
func NewCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
|
||||
return NewFilteredCephNVMeOFGatewayInformer(client, namespace, resyncPeriod, indexers, nil)
|
||||
}
|
||||
|
||||
// NewFilteredCephNVMeOFGatewayInformer constructs a new informer for CephNVMeOFGateway type.
|
||||
// Always prefer using an informer factory to get a shared informer instead of getting an independent
|
||||
// one. This reduces memory footprint and number of connections to the server.
|
||||
func NewFilteredCephNVMeOFGatewayInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
|
||||
return cache.NewSharedIndexInformer(
|
||||
&cache.ListWatch{
|
||||
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
|
||||
if tweakListOptions != nil {
|
||||
tweakListOptions(&options)
|
||||
}
|
||||
return client.CephV1().CephNVMeOFGateways(namespace).List(context.Background(), options)
|
||||
},
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
if tweakListOptions != nil {
|
||||
tweakListOptions(&options)
|
||||
}
|
||||
return client.CephV1().CephNVMeOFGateways(namespace).Watch(context.Background(), options)
|
||||
},
|
||||
ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) {
|
||||
if tweakListOptions != nil {
|
||||
tweakListOptions(&options)
|
||||
}
|
||||
return client.CephV1().CephNVMeOFGateways(namespace).List(ctx, options)
|
||||
},
|
||||
WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) {
|
||||
if tweakListOptions != nil {
|
||||
tweakListOptions(&options)
|
||||
}
|
||||
return client.CephV1().CephNVMeOFGateways(namespace).Watch(ctx, options)
|
||||
},
|
||||
},
|
||||
&apiscephrookiov1.CephNVMeOFGateway{},
|
||||
resyncPeriod,
|
||||
indexers,
|
||||
)
|
||||
}
|
||||
|
||||
func (f *cephNVMeOFGatewayInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
|
||||
return NewFilteredCephNVMeOFGatewayInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
|
||||
}
|
||||
|
||||
func (f *cephNVMeOFGatewayInformer) Informer() cache.SharedIndexInformer {
|
||||
return f.factory.InformerFor(&apiscephrookiov1.CephNVMeOFGateway{}, f.defaultInformer)
|
||||
}
|
||||
|
||||
func (f *cephNVMeOFGatewayInformer) Lister() cephrookiov1.CephNVMeOFGatewayLister {
|
||||
return cephrookiov1.NewCephNVMeOFGatewayLister(f.Informer().GetIndexer())
|
||||
}
|
||||
@@ -46,6 +46,8 @@ type Interface interface {
|
||||
CephFilesystemSubVolumeGroups() CephFilesystemSubVolumeGroupInformer
|
||||
// CephNFSes returns a CephNFSInformer.
|
||||
CephNFSes() CephNFSInformer
|
||||
// CephNVMeOFGateways returns a CephNVMeOFGatewayInformer.
|
||||
CephNVMeOFGateways() CephNVMeOFGatewayInformer
|
||||
// CephObjectRealms returns a CephObjectRealmInformer.
|
||||
CephObjectRealms() CephObjectRealmInformer
|
||||
// CephObjectStores returns a CephObjectStoreInformer.
|
||||
@@ -126,6 +128,11 @@ func (v *version) CephNFSes() CephNFSInformer {
|
||||
return &cephNFSInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
||||
}
|
||||
|
||||
// CephNVMeOFGateways returns a CephNVMeOFGatewayInformer.
|
||||
func (v *version) CephNVMeOFGateways() CephNVMeOFGatewayInformer {
|
||||
return &cephNVMeOFGatewayInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
||||
}
|
||||
|
||||
// CephObjectRealms returns a CephObjectRealmInformer.
|
||||
func (v *version) CephObjectRealms() CephObjectRealmInformer {
|
||||
return &cephObjectRealmInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
|
||||
|
||||
@@ -75,6 +75,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
|
||||
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephFilesystemSubVolumeGroups().Informer()}, nil
|
||||
case v1.SchemeGroupVersion.WithResource("cephnfses"):
|
||||
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNFSes().Informer()}, nil
|
||||
case v1.SchemeGroupVersion.WithResource("cephnvmeofgateways"):
|
||||
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephNVMeOFGateways().Informer()}, nil
|
||||
case v1.SchemeGroupVersion.WithResource("cephobjectrealms"):
|
||||
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephObjectRealms().Informer()}, nil
|
||||
case v1.SchemeGroupVersion.WithResource("cephobjectstores"):
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by lister-gen. DO NOT EDIT.
|
||||
|
||||
package v1
|
||||
|
||||
import (
|
||||
cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
labels "k8s.io/apimachinery/pkg/labels"
|
||||
listers "k8s.io/client-go/listers"
|
||||
cache "k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
// CephNVMeOFGatewayLister helps list CephNVMeOFGateways.
|
||||
// All objects returned here must be treated as read-only.
|
||||
type CephNVMeOFGatewayLister interface {
|
||||
// List lists all CephNVMeOFGateways in the indexer.
|
||||
// Objects returned here must be treated as read-only.
|
||||
List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error)
|
||||
// CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways.
|
||||
CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister
|
||||
CephNVMeOFGatewayListerExpansion
|
||||
}
|
||||
|
||||
// cephNVMeOFGatewayLister implements the CephNVMeOFGatewayLister interface.
|
||||
type cephNVMeOFGatewayLister struct {
|
||||
listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway]
|
||||
}
|
||||
|
||||
// NewCephNVMeOFGatewayLister returns a new CephNVMeOFGatewayLister.
|
||||
func NewCephNVMeOFGatewayLister(indexer cache.Indexer) CephNVMeOFGatewayLister {
|
||||
return &cephNVMeOFGatewayLister{listers.New[*cephrookiov1.CephNVMeOFGateway](indexer, cephrookiov1.Resource("cephnvmeofgateway"))}
|
||||
}
|
||||
|
||||
// CephNVMeOFGateways returns an object that can list and get CephNVMeOFGateways.
|
||||
func (s *cephNVMeOFGatewayLister) CephNVMeOFGateways(namespace string) CephNVMeOFGatewayNamespaceLister {
|
||||
return cephNVMeOFGatewayNamespaceLister{listers.NewNamespaced[*cephrookiov1.CephNVMeOFGateway](s.ResourceIndexer, namespace)}
|
||||
}
|
||||
|
||||
// CephNVMeOFGatewayNamespaceLister helps list and get CephNVMeOFGateways.
|
||||
// All objects returned here must be treated as read-only.
|
||||
type CephNVMeOFGatewayNamespaceLister interface {
|
||||
// List lists all CephNVMeOFGateways in the indexer for a given namespace.
|
||||
// Objects returned here must be treated as read-only.
|
||||
List(selector labels.Selector) (ret []*cephrookiov1.CephNVMeOFGateway, err error)
|
||||
// Get retrieves the CephNVMeOFGateway from the indexer for a given namespace and name.
|
||||
// Objects returned here must be treated as read-only.
|
||||
Get(name string) (*cephrookiov1.CephNVMeOFGateway, error)
|
||||
CephNVMeOFGatewayNamespaceListerExpansion
|
||||
}
|
||||
|
||||
// cephNVMeOFGatewayNamespaceLister implements the CephNVMeOFGatewayNamespaceLister
|
||||
// interface.
|
||||
type cephNVMeOFGatewayNamespaceLister struct {
|
||||
listers.ResourceIndexer[*cephrookiov1.CephNVMeOFGateway]
|
||||
}
|
||||
@@ -106,6 +106,14 @@ type CephNFSListerExpansion interface{}
|
||||
// CephNFSNamespaceLister.
|
||||
type CephNFSNamespaceListerExpansion interface{}
|
||||
|
||||
// CephNVMeOFGatewayListerExpansion allows custom methods to be added to
|
||||
// CephNVMeOFGatewayLister.
|
||||
type CephNVMeOFGatewayListerExpansion interface{}
|
||||
|
||||
// CephNVMeOFGatewayNamespaceListerExpansion allows custom methods to be added to
|
||||
// CephNVMeOFGatewayNamespaceLister.
|
||||
type CephNVMeOFGatewayNamespaceListerExpansion interface{}
|
||||
|
||||
// CephObjectRealmListerExpansion allows custom methods to be added to
|
||||
// CephObjectRealmLister.
|
||||
type CephObjectRealmListerExpansion interface{}
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/file/mirror"
|
||||
"github.com/rook/rook/pkg/operator/ceph/file/subvolumegroup"
|
||||
"github.com/rook/rook/pkg/operator/ceph/nfs"
|
||||
"github.com/rook/rook/pkg/operator/ceph/nvmeof"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/bucket"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/cosi"
|
||||
@@ -82,6 +83,7 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte
|
||||
nfs.Add,
|
||||
rbd.Add,
|
||||
client.Add,
|
||||
nvmeof.Add,
|
||||
mirror.Add,
|
||||
Add,
|
||||
csi.Add,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -xEeuo pipefail
|
||||
|
||||
cat << EOF > /etc/ceph/ceph.conf
|
||||
[global]
|
||||
mon_host = ${ROOK_CEPH_MON_HOST}
|
||||
log_to_stderr = true
|
||||
keyring = /etc/ceph/keyring
|
||||
EOF
|
||||
|
||||
cp /tmp/ceph/keyring /etc/ceph
|
||||
|
||||
chmod 444 /etc/ceph/ceph.conf
|
||||
chmod 440 /etc/ceph/keyring
|
||||
|
||||
# ARGS: all args are assumed to be 'ceph' CLI args
|
||||
# Use "$@" to pass through Ceph CLI arguments (e.g., --connect-timeout, --mon-host, --keyring, etc.)
|
||||
# This allows easy passthru of arguments set by Rook's methods for Ceph CLI options
|
||||
|
||||
sed -e "s/@@POD_NAME@@/${GATEWAY_NAME}/g" \
|
||||
-e "s/@@ANA_GROUP@@/${ANA_GROUP}/g" \
|
||||
-e "s/@@POD_IP@@/${POD_IP}/g" \
|
||||
< /config/nvmeof.conf > /etc/ceph/nvmeof.conf
|
||||
|
||||
ceph "$@" nvme-gw create "${GATEWAY_NAME}" "${POOL_NAME}" "${ANA_GROUP}" || true
|
||||
ceph "$@" nvme-gw show ${POOL_NAME} ${ANA_GROUP} || true
|
||||
@@ -0,0 +1,587 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package nvmeof
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
csiopv1 "github.com/ceph/ceph-csi-operator/api/v1"
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
addonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/ceph/reporting"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"gopkg.in/ini.v1"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/client-go/util/retry"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/manager"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
"sigs.k8s.io/controller-runtime/pkg/source"
|
||||
)
|
||||
|
||||
const (
|
||||
packageName = "ceph-nvmeof-gateway"
|
||||
controllerName = packageName + "-controller"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", packageName)
|
||||
|
||||
var objectsToWatch = []client.Object{
|
||||
&v1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: v1.SchemeGroupVersion.String()}},
|
||||
&appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: appsv1.SchemeGroupVersion.String()}},
|
||||
}
|
||||
|
||||
var controllerTypeMeta = metav1.TypeMeta{
|
||||
Kind: reflect.TypeFor[cephv1.CephNVMeOFGateway]().Name(),
|
||||
APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version),
|
||||
}
|
||||
|
||||
var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
|
||||
|
||||
type ReconcileCephNVMeOFGateway struct {
|
||||
client client.Client
|
||||
scheme *runtime.Scheme
|
||||
context *clusterd.Context
|
||||
cephClusterSpec *cephv1.ClusterSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
opManagerContext context.Context
|
||||
opConfig opcontroller.OperatorConfig
|
||||
recorder record.EventRecorder
|
||||
shouldRotateCephxKeys bool
|
||||
}
|
||||
|
||||
// Add creates a new CephNVMeOFGateway Controller and adds it to the Manager.
|
||||
func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
|
||||
return add(mgr, newReconciler(mgr, context, opManagerContext, opConfig))
|
||||
}
|
||||
|
||||
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
|
||||
return &ReconcileCephNVMeOFGateway{
|
||||
client: mgr.GetClient(),
|
||||
scheme: mgr.GetScheme(),
|
||||
context: context,
|
||||
opManagerContext: opManagerContext,
|
||||
opConfig: opConfig,
|
||||
recorder: mgr.GetEventRecorderFor("rook-" + controllerName),
|
||||
}
|
||||
}
|
||||
|
||||
func watchOwnedCoreObject[T client.Object](c controller.Controller, mgr manager.Manager, obj T) error {
|
||||
return c.Watch(
|
||||
source.Kind(
|
||||
mgr.GetCache(),
|
||||
obj,
|
||||
handler.TypedEnqueueRequestForOwner[T](
|
||||
mgr.GetScheme(),
|
||||
mgr.GetRESTMapper(),
|
||||
&cephv1.CephNVMeOFGateway{},
|
||||
),
|
||||
opcontroller.WatchPredicateForNonCRDObject[T](&cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta}, mgr.GetScheme()),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func add(mgr manager.Manager, r reconcile.Reconciler) error {
|
||||
c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create controller")
|
||||
}
|
||||
logger.Info("successfully started")
|
||||
|
||||
err = addonsv1alpha1.AddToScheme(mgr.GetScheme())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to add addonsv1alpha1 to scheme")
|
||||
}
|
||||
|
||||
err = csiopv1.AddToScheme(mgr.GetScheme())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to add csiopv1 to scheme")
|
||||
}
|
||||
|
||||
err = c.Watch(
|
||||
source.Kind(
|
||||
mgr.GetCache(),
|
||||
&cephv1.CephNVMeOFGateway{TypeMeta: controllerTypeMeta},
|
||||
&handler.TypedEnqueueRequestForObject[*cephv1.CephNVMeOFGateway]{},
|
||||
opcontroller.WatchControllerPredicate[*cephv1.CephNVMeOFGateway](mgr.GetScheme()),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to watch CephNVMeOFGateway CRD")
|
||||
}
|
||||
|
||||
for _, t := range objectsToWatch {
|
||||
err = watchOwnedCoreObject(c, mgr, t)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to watch resource type %s", t.GetObjectKind().GroupVersionKind().String())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reconcile reads the state of the cluster for a CephNVMeOFGateway object and makes changes based on the state read.
|
||||
func (r *ReconcileCephNVMeOFGateway) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
|
||||
defer opcontroller.RecoverAndLogException()
|
||||
reconcileResponse, cephNVMeOFGateway, err := r.reconcile(request)
|
||||
result, reportErr := reporting.ReportReconcileResult(logger, r.recorder, request, &cephNVMeOFGateway, reconcileResponse, err)
|
||||
return result, reportErr
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) reconcile(request reconcile.Request) (reconcile.Result, cephv1.CephNVMeOFGateway, error) {
|
||||
cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{}
|
||||
err := r.client.Get(r.opManagerContext, request.NamespacedName, cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, nil
|
||||
}
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to get CephNVMeOFGateway")
|
||||
}
|
||||
|
||||
observedGeneration := cephNVMeOFGateway.ObjectMeta.Generation
|
||||
|
||||
generationUpdated, err := opcontroller.AddFinalizerIfNotPresent(r.opManagerContext, r.client, cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to add finalizer")
|
||||
}
|
||||
if generationUpdated {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, nil
|
||||
}
|
||||
|
||||
if cephNVMeOFGateway.Status == nil {
|
||||
cephxUninitialized := keyring.UninitializedCephxStatus()
|
||||
err := r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, &cephxUninitialized, k8sutil.EmptyStatus)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed set empty status")
|
||||
}
|
||||
cephNVMeOFGateway.Status = &cephv1.NVMeOFGatewayStatus{
|
||||
Status: cephv1.Status{},
|
||||
Cephx: cephv1.LocalCephxStatus{Daemon: cephxUninitialized},
|
||||
}
|
||||
}
|
||||
|
||||
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
|
||||
|
||||
if !isReadyToReconcile {
|
||||
if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() && !cephClusterExists {
|
||||
err := opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer")
|
||||
}
|
||||
r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer")
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, nil
|
||||
}
|
||||
return reconcileResponse, *cephNVMeOFGateway, nil
|
||||
}
|
||||
r.cephClusterSpec = &cephCluster.Spec
|
||||
|
||||
r.clusterInfo, _, _, err = opcontroller.LoadClusterInfo(r.context, r.opManagerContext, request.NamespacedName.Namespace, r.cephClusterSpec)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to populate cluster info")
|
||||
}
|
||||
|
||||
if !cephNVMeOFGateway.GetDeletionTimestamp().IsZero() {
|
||||
logger.Infof("deleting ceph nvmeof gateway %q", cephNVMeOFGateway.Name)
|
||||
r.recorder.Eventf(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileStarted), "deleting CephNVMeOFGateway %q", cephNVMeOFGateway.Name)
|
||||
|
||||
runningCephVersion, err := cephclient.LeastUptodateDaemonVersion(r.context, r.clusterInfo, config.MonType)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "failed to retrieve current ceph version")
|
||||
}
|
||||
r.clusterInfo.CephVersion = runningCephVersion
|
||||
|
||||
err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to remove finalizer")
|
||||
}
|
||||
r.recorder.Event(cephNVMeOFGateway, v1.EventTypeNormal, string(cephv1.ReconcileSucceeded), "successfully removed finalizer")
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, nil
|
||||
}
|
||||
|
||||
runningCephVersion, desiredCephVersion, err := currentAndDesiredCephVersion(
|
||||
r.opManagerContext, r.opConfig.Image, cephNVMeOFGateway.Namespace, controllerName,
|
||||
k8sutil.NewOwnerInfo(cephNVMeOFGateway, r.scheme), r.context, r.cephClusterSpec, r.clusterInfo,
|
||||
)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), opcontroller.UninitializedCephConfigError) {
|
||||
logger.Info(opcontroller.OperatorNotInitializedMessage)
|
||||
return opcontroller.WaitForRequeueIfOperatorNotInitialized, *cephNVMeOFGateway, nil
|
||||
}
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to detect ceph version")
|
||||
}
|
||||
|
||||
if !cephCluster.Spec.External.Enable && !reflect.DeepEqual(*runningCephVersion, *desiredCephVersion) {
|
||||
return opcontroller.WaitForRequeueIfCephClusterIsUpgrading, *cephNVMeOFGateway,
|
||||
opcontroller.ErrorCephUpgradingRequeue(desiredCephVersion, runningCephVersion)
|
||||
}
|
||||
r.clusterInfo.CephVersion = *runningCephVersion
|
||||
|
||||
if err := validateGateway(cephNVMeOFGateway); err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrapf(err, "invalid configuration")
|
||||
}
|
||||
|
||||
r.shouldRotateCephxKeys, err = keyring.ShouldRotateCephxKeys(cephCluster.Spec.Security.CephX.Daemon, *runningCephVersion,
|
||||
*desiredCephVersion, cephNVMeOFGateway.Status.Cephx.Daemon)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to determine if cephx keys should be rotated")
|
||||
}
|
||||
if r.shouldRotateCephxKeys {
|
||||
logger.Infof("cephx keys will be rotated for %q", request.NamespacedName)
|
||||
}
|
||||
|
||||
_, err = r.reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, errors.Wrap(err, "failed to create deployments")
|
||||
}
|
||||
|
||||
cephxStatus := keyring.UpdatedCephxStatus(r.shouldRotateCephxKeys, cephCluster.Spec.Security.CephX.Daemon, r.clusterInfo.CephVersion, cephNVMeOFGateway.Status.Cephx.Daemon)
|
||||
err = r.updateStatus(observedGeneration, request.NamespacedName, &cephxStatus, k8sutil.ReadyStatus)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to update status: %v", err)
|
||||
return opcontroller.ImmediateRetryResult, *cephNVMeOFGateway, errors.Wrapf(err, "failed to update status")
|
||||
}
|
||||
|
||||
return reconcile.Result{}, *cephNVMeOFGateway, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) reconcileCreateCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) (reconcile.Result, error) {
|
||||
if r.cephClusterSpec.External.Enable {
|
||||
_, err := opcontroller.ValidateCephVersionsBetweenLocalAndExternalClusters(r.context, r.clusterInfo)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "refusing to run new crd")
|
||||
}
|
||||
}
|
||||
|
||||
listOps := metav1.ListOptions{
|
||||
LabelSelector: fmt.Sprintf("%s=%s,app.kubernetes.io/part-of=%s", k8sutil.AppAttr, AppName, cephNVMeOFGateway.Name),
|
||||
}
|
||||
deployments, err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).List(r.opManagerContext, listOps)
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to list deployments")
|
||||
}
|
||||
|
||||
currentGatewayCount := 0
|
||||
if deployments != nil {
|
||||
currentGatewayCount = len(deployments.Items)
|
||||
}
|
||||
|
||||
if currentGatewayCount > cephNVMeOFGateway.Spec.Instances {
|
||||
logger.Infof("scaling down from %d to %d", currentGatewayCount, cephNVMeOFGateway.Spec.Instances)
|
||||
err := r.downCephNVMeOFGateway(cephNVMeOFGateway, currentGatewayCount)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to scale down")
|
||||
}
|
||||
}
|
||||
|
||||
err = r.upCephNVMeOFGateway(cephNVMeOFGateway)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, errors.Wrapf(err, "failed to update gateway")
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) upCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway) error {
|
||||
for i := 0; i < cephNVMeOFGateway.Spec.Instances; i++ {
|
||||
daemonID := k8sutil.IndexToName(i)
|
||||
|
||||
var configMapName, configHash string
|
||||
var err error
|
||||
|
||||
if cephNVMeOFGateway.Spec.ConfigMapRef == "" {
|
||||
configMapName, configHash, err = r.createConfigMap(cephNVMeOFGateway, daemonID)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create configmap for %q", daemonID)
|
||||
}
|
||||
logger.Infof("configmap %q created/updated for nvmeof gateway %q instance %q with hash %q", configMapName, cephNVMeOFGateway.Name, daemonID, configHash)
|
||||
} else {
|
||||
configMapName = cephNVMeOFGateway.Spec.ConfigMapRef
|
||||
configHash = "" // Will be empty for custom configmap
|
||||
}
|
||||
|
||||
deployment, err := r.makeDeployment(cephNVMeOFGateway, daemonID, configMapName, configHash)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to make deployment for %q", daemonID)
|
||||
}
|
||||
|
||||
err = controllerutil.SetControllerReference(cephNVMeOFGateway, deployment, r.scheme)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set owner reference for deployment %q", deployment.Name)
|
||||
}
|
||||
|
||||
_, err = k8sutil.CreateOrUpdateDeployment(r.opManagerContext, r.context.Clientset, deployment)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create/update deployment for %q", daemonID)
|
||||
}
|
||||
|
||||
err = r.createCephNVMeOFService(cephNVMeOFGateway, daemonID)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create service for %q", daemonID)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) downCephNVMeOFGateway(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, currentCount int) error {
|
||||
for i := cephNVMeOFGateway.Spec.Instances; i < currentCount; i++ {
|
||||
daemonID := k8sutil.IndexToName(i)
|
||||
name := instanceName(cephNVMeOFGateway, daemonID)
|
||||
|
||||
err := r.context.Clientset.AppsV1().Deployments(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "failed to delete deployment %q", name)
|
||||
}
|
||||
|
||||
err = r.context.Clientset.CoreV1().Services(cephNVMeOFGateway.Namespace).Delete(r.opManagerContext, name, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
return errors.Wrapf(err, "failed to delete service %q", name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getNVMeOFGatewayConfig generates a complete nvmeof.conf configuration file
|
||||
// with all values filled in (no placeholders). User overrides from nvmeofConfig
|
||||
// are merged on top of the default configuration.
|
||||
func getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup string, userConfig map[string]map[string]string) (string, error) {
|
||||
cfg := ini.Empty()
|
||||
// Set default [gateway] section
|
||||
gatewaySection, err := cfg.NewSection("gateway")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create gateway section")
|
||||
}
|
||||
gatewaySection.Key("name").SetValue(podName)
|
||||
gatewaySection.Key("group").SetValue(anaGroup)
|
||||
gatewaySection.Key("addr").SetValue(podIP)
|
||||
gatewaySection.Key("port").SetValue("5500")
|
||||
gatewaySection.Key("enable_auth").SetValue("False")
|
||||
gatewaySection.Key("state_update_notify").SetValue("True")
|
||||
gatewaySection.Key("state_update_timeout_in_msec").SetValue("2000")
|
||||
gatewaySection.Key("state_update_interval_sec").SetValue("5")
|
||||
gatewaySection.Key("enable_spdk_discovery_controller").SetValue("False")
|
||||
gatewaySection.Key("encryption_key").SetValue("/etc/ceph/encryption.key")
|
||||
gatewaySection.Key("rebalance_period_sec").SetValue("7")
|
||||
gatewaySection.Key("max_gws_in_grp").SetValue("16")
|
||||
gatewaySection.Key("max_ns_to_change_lb_grp").SetValue("8")
|
||||
gatewaySection.Key("verify_listener_ip").SetValue("False")
|
||||
gatewaySection.Key("enable_monitor_client").SetValue("True")
|
||||
|
||||
// Set default [discovery] section
|
||||
discoverySection, err := cfg.NewSection("discovery")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create discovery section")
|
||||
}
|
||||
discoverySection.Key("addr").SetValue("0.0.0.0")
|
||||
discoverySection.Key("port").SetValue("8009")
|
||||
|
||||
// Set default [ceph] section
|
||||
cephSection, err := cfg.NewSection("ceph")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create ceph section")
|
||||
}
|
||||
cephSection.Key("id").SetValue("admin")
|
||||
cephSection.Key("pool").SetValue(poolName)
|
||||
cephSection.Key("config_file").SetValue("/etc/ceph/ceph.conf")
|
||||
|
||||
// Set default [mtls] section
|
||||
mtlsSection, err := cfg.NewSection("mtls")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create mtls section")
|
||||
}
|
||||
mtlsSection.Key("server_key").SetValue("./server.key")
|
||||
mtlsSection.Key("client_key").SetValue("./client.key")
|
||||
mtlsSection.Key("server_cert").SetValue("./server.crt")
|
||||
mtlsSection.Key("client_cert").SetValue("./client.crt")
|
||||
|
||||
// Set default [spdk] section
|
||||
spdkSection, err := cfg.NewSection("spdk")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create spdk section")
|
||||
}
|
||||
spdkSection.Key("bdevs_per_cluster").SetValue("32")
|
||||
spdkSection.Key("mem_size").SetValue("4096")
|
||||
spdkSection.Key("tgt_path").SetValue("/usr/local/bin/nvmf_tgt")
|
||||
spdkSection.Key("timeout").SetValue("60.0")
|
||||
spdkSection.Key("rpc_socket").SetValue("/var/tmp/spdk.sock")
|
||||
|
||||
// Set default [monitor] section
|
||||
monitorSection, err := cfg.NewSection("monitor")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create monitor section")
|
||||
}
|
||||
monitorSection.Key("port").SetValue("5499")
|
||||
|
||||
// Apply user overrides
|
||||
for sectionName, options := range userConfig {
|
||||
section := cfg.Section(sectionName)
|
||||
if section == nil {
|
||||
// Section doesn't exist, create it
|
||||
var createErr error
|
||||
section, createErr = cfg.NewSection(sectionName)
|
||||
if createErr != nil {
|
||||
return "", errors.Wrapf(createErr, "failed to create section %q", sectionName)
|
||||
}
|
||||
}
|
||||
for key, value := range options {
|
||||
section.Key(key).SetValue(value)
|
||||
}
|
||||
}
|
||||
|
||||
// Write to string with proper formatting
|
||||
var buf strings.Builder
|
||||
_, err = cfg.WriteTo(&buf)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to write config to string")
|
||||
}
|
||||
|
||||
return buf.String(), nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) generateConfigMap(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) (*v1.ConfigMap, error) {
|
||||
poolName := nvmeof.Spec.Pool
|
||||
anaGroup := nvmeof.Spec.Group
|
||||
podName := instanceName(nvmeof, daemonID)
|
||||
// Use placeholder that will be replaced at runtime with actual pod IP
|
||||
// The init container will replace @@POD_IP@@ with the actual pod IP
|
||||
podIP := "@@POD_IP@@"
|
||||
|
||||
configContent, err := getNVMeOFGatewayConfig(poolName, podName, podIP, anaGroup, nvmeof.Spec.NVMeOFConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to generate nvmeof config")
|
||||
}
|
||||
data := map[string]string{
|
||||
"config": configContent,
|
||||
}
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-%s-config", nvmeof.Name, daemonID)
|
||||
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: configMapName,
|
||||
Namespace: nvmeof.Namespace,
|
||||
Labels: getLabels(nvmeof, daemonID),
|
||||
},
|
||||
Data: data,
|
||||
}
|
||||
|
||||
return configMap, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) createConfigMap(cephNVMeOFGateway *cephv1.CephNVMeOFGateway, daemonID string) (string, string, error) {
|
||||
configMap, err := r.generateConfigMap(cephNVMeOFGateway, daemonID)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to generate configmap: %v", err)
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
err = controllerutil.SetControllerReference(cephNVMeOFGateway, configMap, r.scheme)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to set owner reference: %v", err)
|
||||
return "", "", errors.Wrapf(err, "failed to set owner reference for nvmeof configmap %q", configMap.Name)
|
||||
}
|
||||
|
||||
if _, err := r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Create(r.opManagerContext, configMap, metav1.CreateOptions{}); err != nil {
|
||||
if !kerrors.IsAlreadyExists(err) {
|
||||
logger.Errorf("failed to create configmap: %v", err)
|
||||
return "", "", errors.Wrap(err, "failed to create nvmeof config map")
|
||||
}
|
||||
|
||||
if _, err = r.context.Clientset.CoreV1().ConfigMaps(cephNVMeOFGateway.Namespace).Update(r.opManagerContext, configMap, metav1.UpdateOptions{}); err != nil {
|
||||
logger.Errorf("failed to update configmap: %v", err)
|
||||
return "", "", errors.Wrap(err, "failed to update nvmeof config map")
|
||||
}
|
||||
}
|
||||
|
||||
configHash := k8sutil.Hash(fmt.Sprintf("%v", configMap.Data))
|
||||
return configMap.Name, configHash, nil
|
||||
}
|
||||
|
||||
func validateGateway(g *cephv1.CephNVMeOFGateway) error {
|
||||
if g.Spec.Instances < 1 {
|
||||
return errors.New("at least one gateway instance is required")
|
||||
}
|
||||
|
||||
if g.Spec.Group == "" {
|
||||
return errors.New("gateway group name is required")
|
||||
}
|
||||
|
||||
if g.Spec.Pool == "" {
|
||||
return errors.New("pool name is required")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) updateStatus(observedGeneration int64, namespacedName types.NamespacedName, cephxStatus *cephv1.CephxStatus, status string) error {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{}
|
||||
|
||||
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
err := r.client.Get(r.opManagerContext, namespacedName, nvmeof)
|
||||
if err != nil {
|
||||
if kerrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
logger.Errorf("failed to get gateway: %v", err)
|
||||
return errors.Wrapf(err, "failed to get for status update")
|
||||
}
|
||||
|
||||
if nvmeof.Status == nil {
|
||||
nvmeof.Status = &cephv1.NVMeOFGatewayStatus{}
|
||||
}
|
||||
|
||||
nvmeof.Status.Phase = status
|
||||
|
||||
if observedGeneration != k8sutil.ObservedGenerationNotAvailable {
|
||||
nvmeof.Status.ObservedGeneration = observedGeneration
|
||||
}
|
||||
|
||||
if cephxStatus != nil {
|
||||
nvmeof.Status.Cephx.Daemon = *cephxStatus
|
||||
}
|
||||
|
||||
if err := reporting.UpdateStatus(r.client, nvmeof); err != nil {
|
||||
return errors.Wrapf(err, "failed to set status")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Errorf("retry loop failed: %v", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,712 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package nvmeof
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
ini "gopkg.in/ini.v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
)
|
||||
|
||||
var (
|
||||
name = "my-nvmeof"
|
||||
namespace = "rook-ceph"
|
||||
)
|
||||
|
||||
func TestCephNVMeOFGatewayController(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
// Set DEBUG logging
|
||||
capnslog.SetGlobalLogLevel(capnslog.DEBUG)
|
||||
os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
||||
|
||||
// Register operator types with the runtime scheme.
|
||||
testScheme := scheme.Scheme
|
||||
testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{})
|
||||
testScheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
||||
|
||||
baseExecutor := func() *exectest.MockExecutor {
|
||||
return &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
||||
logger.Infof("mock execute: %s %v", command, args)
|
||||
if command == "ceph" && args[0] == "status" {
|
||||
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
||||
}
|
||||
panic(fmt.Sprintf("unhandled command %s %v", command, args))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
successExecutor := func(t *testing.T) *exectest.MockExecutor {
|
||||
t.Helper()
|
||||
|
||||
return &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
||||
logger.Infof("mock execute: %s %v", command, args)
|
||||
if command == "ceph" {
|
||||
if args[0] == "status" {
|
||||
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
||||
}
|
||||
if args[0] == "nvme-gw" && args[1] == "create" {
|
||||
return "", nil
|
||||
}
|
||||
if args[0] == "nvme-gw" && args[1] == "show" {
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
panic(fmt.Sprintf("unhandled command %s %v", command, args))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
newContext := func(executor *exectest.MockExecutor) *clusterd.Context {
|
||||
clientset := test.New(t, 3)
|
||||
return &clusterd.Context{
|
||||
Executor: executor,
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
}
|
||||
|
||||
baseCephNVMeOFGateway := func() *cephv1.CephNVMeOFGateway {
|
||||
return &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Finalizers: []string{"cephnvmeofgateway.ceph.rook.io"},
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
}
|
||||
}
|
||||
|
||||
cephClusterNotReady := func() *cephv1.CephCluster {
|
||||
return &cephv1.CephCluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: namespace,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Status: cephv1.ClusterStatus{
|
||||
Phase: "",
|
||||
CephStatus: &cephv1.CephStatus{
|
||||
Health: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
cephClusterReady := func(clusterCtx *clusterd.Context) *cephv1.CephCluster {
|
||||
cephCluster := cephClusterNotReady()
|
||||
cephCluster.Status.Phase = k8sutil.ReadyStatus
|
||||
cephCluster.Status.CephStatus.Health = "HEALTH_OK"
|
||||
|
||||
secrets := map[string][]byte{
|
||||
"fsid": []byte(name),
|
||||
"mon-secret": []byte("monsecret"),
|
||||
"admin-secret": []byte("adminsecret"),
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-mon",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
_, err := clusterCtx.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("should be no error: %v", err))
|
||||
}
|
||||
|
||||
return cephCluster
|
||||
}
|
||||
|
||||
newControllerClient := func(objects ...runtime.Object) client.WithWatch {
|
||||
return fake.NewClientBuilder().WithScheme(testScheme).WithRuntimeObjects(objects...).Build()
|
||||
}
|
||||
|
||||
newReconcile := func(clusterCtx *clusterd.Context, cl client.WithWatch) *ReconcileCephNVMeOFGateway {
|
||||
return &ReconcileCephNVMeOFGateway{client: cl, scheme: testScheme, context: clusterCtx, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
|
||||
}
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
|
||||
currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
|
||||
return &version.Squid, &version.Squid, nil
|
||||
}
|
||||
|
||||
t.Run("error - no ceph cluster", func(t *testing.T) {
|
||||
cCtx := newContext(baseExecutor())
|
||||
cl := newControllerClient(baseCephNVMeOFGateway())
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, res.RequeueAfter, time.Duration(0))
|
||||
})
|
||||
|
||||
t.Run("error - ceph cluster not ready", func(t *testing.T) {
|
||||
cCtx := newContext(baseExecutor())
|
||||
cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterNotReady())
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, res.RequeueAfter, time.Duration(0))
|
||||
})
|
||||
|
||||
t.Run("error - invalid gateway spec", func(t *testing.T) {
|
||||
t.Run("instances less than 1 should error", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
nvmeof := baseCephNVMeOFGateway()
|
||||
nvmeof.Spec.Instances = 0
|
||||
cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
fakeRecorder := record.NewFakeRecorder(5)
|
||||
r.recorder = fakeRecorder
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assert.Contains(t, err.Error(), "invalid configuration")
|
||||
|
||||
assert.Len(t, fakeRecorder.Events, 1)
|
||||
event := <-fakeRecorder.Events
|
||||
assert.Contains(t, event, "ReconcileFailed")
|
||||
assert.Contains(t, event, "invalid configuration")
|
||||
})
|
||||
|
||||
t.Run("group empty should error", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
nvmeof := baseCephNVMeOFGateway()
|
||||
nvmeof.Spec.Group = ""
|
||||
cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
fakeRecorder := record.NewFakeRecorder(5)
|
||||
r.recorder = fakeRecorder
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assert.Contains(t, err.Error(), "invalid configuration")
|
||||
|
||||
assert.Len(t, fakeRecorder.Events, 1)
|
||||
event := <-fakeRecorder.Events
|
||||
assert.Contains(t, event, "ReconcileFailed")
|
||||
assert.Contains(t, event, "invalid configuration")
|
||||
})
|
||||
})
|
||||
|
||||
assertCephNVMeOFGatewayReady := func(t *testing.T, r *ReconcileCephNVMeOFGateway, names ...string) {
|
||||
t.Helper()
|
||||
|
||||
if len(names) == 0 {
|
||||
names = []string{name}
|
||||
}
|
||||
|
||||
for _, n := range names {
|
||||
cephNVMeOFGateway := &cephv1.CephNVMeOFGateway{}
|
||||
err := r.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: n}, cephNVMeOFGateway)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Ready", cephNVMeOFGateway.Status.Phase, cephNVMeOFGateway)
|
||||
}
|
||||
}
|
||||
|
||||
assertResourcesExist := func(t *testing.T, cCtx *clusterd.Context, names ...string) {
|
||||
t.Helper()
|
||||
|
||||
depNames := []string{}
|
||||
deps, err := cCtx.Clientset.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
for _, dep := range deps.Items {
|
||||
depNames = append(depNames, dep.Name)
|
||||
}
|
||||
assert.ElementsMatch(t, names, depNames)
|
||||
|
||||
svcNames := []string{}
|
||||
svcs, err := cCtx.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{})
|
||||
assert.NoError(t, err)
|
||||
for _, dep := range svcs.Items {
|
||||
svcNames = append(svcNames, dep.Name)
|
||||
}
|
||||
assert.ElementsMatch(t, names, svcNames)
|
||||
}
|
||||
|
||||
t.Run("run one nvmeof gateway", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
cl := newControllerClient(baseCephNVMeOFGateway(), cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
t.Run("initial reconcile", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
|
||||
})
|
||||
|
||||
t.Run("double reconcile", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("run multiple nvmeof gateways", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
nvmeof := baseCephNVMeOFGateway()
|
||||
nvmeof.Spec.Instances = 3
|
||||
cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
t.Run("initial reconcile", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
|
||||
})
|
||||
|
||||
t.Run("double reconcile", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("scale down nvmeof gateways", func(t *testing.T) {
|
||||
t.Run("scale from 3 to 2 gateways", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
nvmeof := baseCephNVMeOFGateway()
|
||||
nvmeof.Spec.Instances = 3
|
||||
cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
|
||||
|
||||
err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof)
|
||||
assert.NoError(t, err)
|
||||
nvmeof.Spec.Instances = 2
|
||||
err = cl.Update(ctx, nvmeof, &client.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
res, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b")
|
||||
})
|
||||
|
||||
t.Run("scale from 3 to 1 gateway", func(t *testing.T) {
|
||||
cCtx := newContext(successExecutor(t))
|
||||
nvmeof := baseCephNVMeOFGateway()
|
||||
nvmeof.Spec.Instances = 3
|
||||
cl := newControllerClient(nvmeof, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
|
||||
|
||||
err = cl.Get(ctx, client.ObjectKeyFromObject(nvmeof), nvmeof)
|
||||
assert.NoError(t, err)
|
||||
nvmeof.Spec.Instances = 1
|
||||
err = cl.Update(ctx, nvmeof, &client.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
res, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r)
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a")
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("multiple CephNVMeOFGateway clusters", func(t *testing.T) {
|
||||
nvmeof1 := baseCephNVMeOFGateway()
|
||||
nvmeof1.Spec.Instances = 3
|
||||
nvmeof2 := baseCephNVMeOFGateway()
|
||||
nvmeof2.Spec.Instances = 2
|
||||
nvmeof2.Name = "nvmeof2"
|
||||
|
||||
cCtx := newContext(successExecutor(t))
|
||||
cl := newControllerClient(nvmeof1, nvmeof2, cephClusterReady(cCtx))
|
||||
r := newReconcile(cCtx, cl)
|
||||
|
||||
req2 := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: "nvmeof2",
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("reconcile first CephNVMeOFGateway cluster", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r, "my-nvmeof") // first cluster should be ready
|
||||
assertResourcesExist(t, cCtx, "rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c")
|
||||
})
|
||||
|
||||
t.Run("reconcile second CephNVMeOFGateway cluster", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req2)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2")
|
||||
assertResourcesExist(t, cCtx,
|
||||
"rook-ceph-nvmeof-my-nvmeof-a", "rook-ceph-nvmeof-my-nvmeof-b", "rook-ceph-nvmeof-my-nvmeof-c",
|
||||
"rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b",
|
||||
)
|
||||
})
|
||||
|
||||
t.Run("scale down first CephNVMeOFGateway cluster (3 to 1)", func(t *testing.T) {
|
||||
err := cl.Get(ctx, client.ObjectKeyFromObject(nvmeof1), nvmeof1)
|
||||
assert.NoError(t, err)
|
||||
nvmeof1.Spec.Instances = 1
|
||||
err = cl.Update(ctx, nvmeof1, &client.UpdateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
|
||||
assertCephNVMeOFGatewayReady(t, r, "my-nvmeof", "nvmeof2")
|
||||
assertResourcesExist(t, cCtx,
|
||||
"rook-ceph-nvmeof-my-nvmeof-a",
|
||||
"rook-ceph-nvmeof-nvmeof2-a", "rook-ceph-nvmeof-nvmeof2-b",
|
||||
)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestNVMeOFKeyRotation(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
var (
|
||||
name = "my-nvmeof"
|
||||
namespace = "rook-ceph"
|
||||
)
|
||||
// Set DEBUG logging
|
||||
t.Setenv("ROOK_LOG_LEVEL", "DEBUG")
|
||||
|
||||
// Mock version to avoid deployment updates (filesystem mirror pattern)
|
||||
currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *cephclient.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) {
|
||||
return &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, &version.CephVersion{Major: 20, Minor: 2, Extra: 0}, nil
|
||||
}
|
||||
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
}
|
||||
|
||||
cephCluster := &cephv1.CephCluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: namespace,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: cephv1.ClusterSpec{
|
||||
Security: cephv1.ClusterSecuritySpec{
|
||||
CephX: cephv1.ClusterCephxConfig{
|
||||
Daemon: cephv1.CephxConfig{},
|
||||
},
|
||||
},
|
||||
},
|
||||
Status: cephv1.ClusterStatus{
|
||||
Phase: k8sutil.ReadyStatus,
|
||||
CephStatus: &cephv1.CephStatus{
|
||||
Health: "HEALTH_OK",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
object := []runtime.Object{
|
||||
nvmeof,
|
||||
cephCluster,
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephNVMeOFGateway{})
|
||||
s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
|
||||
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
||||
if command == "ceph" {
|
||||
if args[0] == "status" {
|
||||
return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_OK"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
|
||||
}
|
||||
if args[0] == "nvme-gw" && args[1] == "create" {
|
||||
return "", nil
|
||||
}
|
||||
if args[0] == "nvme-gw" && args[1] == "show" {
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
},
|
||||
}
|
||||
clientset := test.New(t, 3)
|
||||
c := &clusterd.Context{
|
||||
Executor: executor,
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
|
||||
|
||||
req := reconcile.Request{
|
||||
NamespacedName: types.NamespacedName{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("first reconcile", func(t *testing.T) {
|
||||
secrets := map[string][]byte{
|
||||
"fsid": []byte(name),
|
||||
"mon-secret": []byte("monsecret"),
|
||||
"admin-secret": []byte("adminsecret"),
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-ceph-mon",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
|
||||
assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
|
||||
})
|
||||
|
||||
t.Run("subsequent reconcile - retain cephx status", func(t *testing.T) {
|
||||
r := &ReconcileCephNVMeOFGateway{client: cl, scheme: s, context: c, opManagerContext: ctx, recorder: record.NewFakeRecorder(5)}
|
||||
_, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, uint32(1), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
|
||||
assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
|
||||
})
|
||||
|
||||
t.Run("brownfield reconcile - retain unknown cephx status", func(t *testing.T) {
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err := cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
nvmeofResult.Status.Cephx.Daemon = cephv1.CephxStatus{}
|
||||
err = cl.Update(ctx, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, cephv1.CephxStatus{}, nvmeofResult.Status.Cephx.Daemon)
|
||||
})
|
||||
|
||||
t.Run("rotate key - brownfield unknown status becomes known", func(t *testing.T) {
|
||||
cluster := cephv1.CephCluster{}
|
||||
err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
|
||||
assert.NoError(t, err)
|
||||
cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
|
||||
KeyRotationPolicy: "KeyGeneration",
|
||||
KeyGeneration: 2,
|
||||
}
|
||||
err = cl.Update(ctx, &cluster)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
|
||||
assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
|
||||
})
|
||||
|
||||
t.Run("brownfield reconcile - no further rotation happens", func(t *testing.T) {
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, res.RequeueAfter, time.Duration(0))
|
||||
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, uint32(2), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
|
||||
assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
|
||||
})
|
||||
|
||||
t.Run("rotate key - cephx status updated", func(t *testing.T) {
|
||||
cluster := cephv1.CephCluster{}
|
||||
err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: namespace}, &cluster)
|
||||
assert.NoError(t, err)
|
||||
cluster.Spec.Security.CephX.Daemon = cephv1.CephxConfig{
|
||||
KeyRotationPolicy: "KeyGeneration",
|
||||
KeyGeneration: 3,
|
||||
}
|
||||
err = cl.Update(ctx, &cluster)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
|
||||
nvmeofResult := cephv1.CephNVMeOFGateway{}
|
||||
err = cl.Get(ctx, req.NamespacedName, &nvmeofResult)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, uint32(3), nvmeofResult.Status.Cephx.Daemon.KeyGeneration)
|
||||
assert.Equal(t, "20.2.0-0", nvmeofResult.Status.Cephx.Daemon.KeyCephVersion)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNVMeOFConfigGeneration(t *testing.T) {
|
||||
t.Run("defaults", func(t *testing.T) {
|
||||
config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", nil)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cfg, err := ini.Load([]byte(config))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "pod-a", cfg.Section("gateway").Key("name").String())
|
||||
assert.Equal(t, "ana-a", cfg.Section("gateway").Key("group").String())
|
||||
assert.Equal(t, "10.0.0.1", cfg.Section("gateway").Key("addr").String())
|
||||
assert.Equal(t, "5500", cfg.Section("gateway").Key("port").String())
|
||||
assert.Equal(t, "0.0.0.0", cfg.Section("discovery").Key("addr").String())
|
||||
assert.Equal(t, "8009", cfg.Section("discovery").Key("port").String())
|
||||
assert.Equal(t, "admin", cfg.Section("ceph").Key("id").String())
|
||||
assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String())
|
||||
assert.Equal(t, "4096", cfg.Section("spdk").Key("mem_size").String())
|
||||
assert.Equal(t, "5499", cfg.Section("monitor").Key("port").String())
|
||||
})
|
||||
|
||||
t.Run("overrides and new section", func(t *testing.T) {
|
||||
userConfig := map[string]map[string]string{
|
||||
"gateway": {
|
||||
"port": "6600",
|
||||
"enable_auth": "True",
|
||||
},
|
||||
"ceph": {
|
||||
"pool": "pool-b",
|
||||
},
|
||||
"custom": {
|
||||
"foo": "bar",
|
||||
},
|
||||
}
|
||||
config, err := getNVMeOFGatewayConfig("pool-a", "pod-a", "10.0.0.1", "ana-a", userConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cfg, err := ini.Load([]byte(config))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "6600", cfg.Section("gateway").Key("port").String())
|
||||
assert.Equal(t, "True", cfg.Section("gateway").Key("enable_auth").String())
|
||||
assert.Equal(t, "pool-b", cfg.Section("ceph").Key("pool").String())
|
||||
assert.Equal(t, "bar", cfg.Section("custom").Key("foo").String())
|
||||
})
|
||||
}
|
||||
|
||||
func TestNVMeOFConfigMapGeneration(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Pool: "pool-a",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
NVMeOFConfig: map[string]map[string]string{
|
||||
"gateway": {
|
||||
"port": "5511",
|
||||
},
|
||||
},
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
}
|
||||
|
||||
r := &ReconcileCephNVMeOFGateway{}
|
||||
configMap, err := r.generateConfigMap(nvmeof, "a")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-a-config", configMap.Name)
|
||||
assert.Equal(t, "rook-ceph", configMap.Namespace)
|
||||
assert.Contains(t, configMap.Data, "config")
|
||||
|
||||
cfg, err := ini.Load([]byte(configMap.Data["config"]))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, instanceName(nvmeof, "a"), cfg.Section("gateway").Key("name").String())
|
||||
assert.Equal(t, "@@POD_IP@@", cfg.Section("gateway").Key("addr").String())
|
||||
assert.Equal(t, "group-a", cfg.Section("gateway").Key("group").String())
|
||||
assert.Equal(t, "5511", cfg.Section("gateway").Key("port").String())
|
||||
assert.Equal(t, "pool-a", cfg.Section("ceph").Key("pool").String())
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package nvmeof
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
)
|
||||
|
||||
const (
|
||||
AppName = "rook-ceph-nvmeof"
|
||||
nvmeofIOPort = 4420
|
||||
nvmeofGatewayPort = 5500
|
||||
nvmeofMonitorPort = 5499
|
||||
nvmeofDiscoveryPort = 8009
|
||||
defaultNVMeOFImage = "quay.io/ceph/nvmeof:1.5"
|
||||
configKey = "config"
|
||||
serviceAccountName = "rook-ceph-nvmeof"
|
||||
)
|
||||
|
||||
// getPorts returns the configured ports with defaults
|
||||
func getPorts(nvmeof *cephv1.CephNVMeOFGateway) (ioPort, gatewayPort, monitorPort, discoveryPort int32) {
|
||||
ioPort = nvmeofIOPort
|
||||
gatewayPort = nvmeofGatewayPort
|
||||
monitorPort = nvmeofMonitorPort
|
||||
discoveryPort = nvmeofDiscoveryPort
|
||||
|
||||
if nvmeof.Spec.Ports != nil {
|
||||
if nvmeof.Spec.Ports.IOPort != 0 {
|
||||
ioPort = nvmeof.Spec.Ports.IOPort
|
||||
}
|
||||
if nvmeof.Spec.Ports.GatewayPort != 0 {
|
||||
gatewayPort = nvmeof.Spec.Ports.GatewayPort
|
||||
}
|
||||
if nvmeof.Spec.Ports.MonitorPort != 0 {
|
||||
monitorPort = nvmeof.Spec.Ports.MonitorPort
|
||||
}
|
||||
if nvmeof.Spec.Ports.DiscoveryPort != 0 {
|
||||
discoveryPort = nvmeof.Spec.Ports.DiscoveryPort
|
||||
}
|
||||
}
|
||||
|
||||
return ioPort, gatewayPort, monitorPort, discoveryPort
|
||||
}
|
||||
|
||||
//go:embed connectionconfig.sh
|
||||
var connectionConfigScript string
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) generateCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) *v1.Service {
|
||||
labels := getLabels(nvmeof, daemonID)
|
||||
serviceName := instanceName(nvmeof, daemonID)
|
||||
ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof)
|
||||
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: serviceName,
|
||||
Namespace: nvmeof.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "io",
|
||||
Port: ioPort,
|
||||
TargetPort: intstr.FromInt(int(ioPort)),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "gateway",
|
||||
Port: gatewayPort,
|
||||
TargetPort: intstr.FromInt(int(gatewayPort)),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "monitor",
|
||||
Port: monitorPort,
|
||||
TargetPort: intstr.FromInt(int(monitorPort)),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "discovery",
|
||||
Port: discoveryPort,
|
||||
TargetPort: intstr.FromInt(int(discoveryPort)),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec)
|
||||
if hostNetwork {
|
||||
svc.Spec.ClusterIP = v1.ClusterIPNone
|
||||
}
|
||||
|
||||
return svc
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) createCephNVMeOFService(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) error {
|
||||
s := r.generateCephNVMeOFService(nvmeof, daemonID)
|
||||
|
||||
err := controllerutil.SetControllerReference(nvmeof, s, r.scheme)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to set owner reference: %v", err)
|
||||
return errors.Wrapf(err, "failed to set owner reference to ceph nvmeof gateway %q", s)
|
||||
}
|
||||
|
||||
svc, err := r.context.Clientset.CoreV1().Services(nvmeof.Namespace).Create(r.opManagerContext, s, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if !kerrors.IsAlreadyExists(err) {
|
||||
logger.Errorf("failed to create service: %v", err)
|
||||
return errors.Wrap(err, "failed to create nvmeof gateway service")
|
||||
}
|
||||
logger.Infof("ceph nvmeof gateway service already created")
|
||||
return nil
|
||||
}
|
||||
|
||||
_, _, _, discoveryPort := getPorts(nvmeof)
|
||||
logger.Infof("ceph nvmeof gateway service running at %s:%d", svc.Spec.ClusterIP, discoveryPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) makeDeployment(nvmeof *cephv1.CephNVMeOFGateway, daemonID, configMapName, configHash string) (*apps.Deployment, error) {
|
||||
if r == nil {
|
||||
logger.Errorf("receiver is nil")
|
||||
return nil, errors.New("receiver is nil")
|
||||
}
|
||||
if r.clusterInfo == nil {
|
||||
logger.Errorf("clusterInfo is nil")
|
||||
return nil, errors.New("clusterInfo is nil")
|
||||
}
|
||||
if r.clusterInfo.CephVersion.String() == "" {
|
||||
logger.Errorf("CephVersion is empty")
|
||||
return nil, errors.New("CephVersion is empty")
|
||||
}
|
||||
|
||||
resourceName := instanceName(nvmeof, daemonID)
|
||||
deployment := &apps.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: resourceName,
|
||||
Namespace: nvmeof.Namespace,
|
||||
Labels: getLabels(nvmeof, daemonID),
|
||||
},
|
||||
}
|
||||
|
||||
hostNetwork := nvmeof.IsHostNetwork(r.cephClusterSpec)
|
||||
|
||||
k8sutil.AddRookVersionLabelToDeployment(deployment)
|
||||
controller.AddCephVersionLabelToDeployment(r.clusterInfo.CephVersion, deployment)
|
||||
nvmeof.Spec.Annotations.ApplyToObjectMeta(&deployment.ObjectMeta)
|
||||
nvmeof.Spec.Labels.ApplyToObjectMeta(&deployment.ObjectMeta)
|
||||
|
||||
cephConfigVol, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount()
|
||||
gatewayConfigVol, gatewayConfigMount := gatewayConfigVolumeAndMount(configMapName)
|
||||
|
||||
initContainer := r.createCephConfigInitContainer(nvmeof, daemonID, gatewayConfigMount)
|
||||
daemonContainer := r.daemonContainer(nvmeof, cephConfigMount)
|
||||
|
||||
gatewayName := instanceName(nvmeof, daemonID)
|
||||
podSpec := v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
initContainer,
|
||||
},
|
||||
Containers: []v1.Container{
|
||||
daemonContainer,
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: []v1.Volume{
|
||||
cephConfigVol,
|
||||
adminKeyringVolume(),
|
||||
gatewayConfigVol,
|
||||
},
|
||||
HostNetwork: hostNetwork,
|
||||
PriorityClassName: nvmeof.Spec.PriorityClassName,
|
||||
SecurityContext: &v1.PodSecurityContext{},
|
||||
ServiceAccountName: serviceAccountName,
|
||||
}
|
||||
|
||||
// Ensure a stable hostname inside the pod (does NOT make the pod IP stable).
|
||||
// This matches the gateway instance name used elsewhere by the operator.
|
||||
if errs := validation.IsDNS1123Label(gatewayName); len(errs) == 0 {
|
||||
podSpec.Hostname = gatewayName
|
||||
} else {
|
||||
logger.Warningf("not setting pod hostname %q (must be a DNS-1123 label): %v", gatewayName, errs)
|
||||
}
|
||||
|
||||
k8sutil.AddUnreachableNodeToleration(&podSpec)
|
||||
|
||||
if hostNetwork {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
nvmeof.Spec.Placement.ApplyToPodSpec(&podSpec)
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: resourceName,
|
||||
Labels: getLabels(nvmeof, daemonID),
|
||||
Annotations: map[string]string{
|
||||
"config-hash": configHash,
|
||||
},
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
|
||||
if hostNetwork {
|
||||
podSpec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if r.cephClusterSpec.Network.IsMultus() {
|
||||
if err := k8sutil.ApplyMultus(r.clusterInfo.Namespace, &r.cephClusterSpec.Network, &podTemplateSpec.ObjectMeta); err != nil {
|
||||
logger.Errorf("failed to apply Multus configuration: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
nvmeof.Spec.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
nvmeof.Spec.Labels.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
|
||||
replicas := int32(1)
|
||||
revisionHistoryLimit := controller.RevisionHistoryLimit()
|
||||
deployment.Spec = apps.DeploymentSpec{
|
||||
RevisionHistoryLimit: revisionHistoryLimit,
|
||||
Selector: &metav1.LabelSelector{MatchLabels: getLabels(nvmeof, daemonID)},
|
||||
Template: podTemplateSpec,
|
||||
Replicas: &replicas,
|
||||
}
|
||||
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
// createCephConfigInitContainer creates a minimal init container that generates
|
||||
// /etc/ceph/ceph.conf, copies the admin keyring, and copies nvmeof.conf from the ConfigMap.
|
||||
// This is needed for the gateway to connect to the Ceph cluster.
|
||||
func (r *ReconcileCephNVMeOFGateway) createCephConfigInitContainer(nvmeof *cephv1.CephNVMeOFGateway, daemonID string, gatewayConfigMount v1.VolumeMount) v1.Container {
|
||||
_, cephConfigMount := controller.ConfGeneratedInPodVolumeAndMount()
|
||||
|
||||
cephImage := r.cephClusterSpec.CephVersion.Image
|
||||
imagePullPolicy := controller.GetContainerImagePullPolicy(r.cephClusterSpec.CephVersion.ImagePullPolicy)
|
||||
|
||||
// Build Ceph CLI arguments using Rook's helper functions
|
||||
// Use AdminFlags since the init container uses admin keyring for Ceph commands
|
||||
// Override the keyring path to use /etc/ceph/keyring (where the script copies it)
|
||||
cephArgs := controller.AdminFlags(r.clusterInfo)
|
||||
// Replace the keyring path with the one used by the init container script
|
||||
for i, arg := range cephArgs {
|
||||
if strings.HasPrefix(arg, "--keyring=") {
|
||||
cephArgs[i] = "--keyring=/etc/ceph/keyring"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Use the embedded script and pass Ceph CLI args via "$@"
|
||||
script := connectionConfigScript
|
||||
|
||||
gatewayName := instanceName(nvmeof, daemonID)
|
||||
poolName := nvmeof.Spec.Pool
|
||||
anaGroup := nvmeof.Spec.Group
|
||||
|
||||
// Build environment variables using Rook's helper functions
|
||||
// DaemonEnvVars already includes StoredMonHostEnvVars() which provides ROOK_CEPH_MON_HOST
|
||||
envVars := controller.DaemonEnvVars(r.cephClusterSpec)
|
||||
envVars = append(envVars,
|
||||
v1.EnvVar{
|
||||
Name: "GATEWAY_NAME",
|
||||
Value: gatewayName,
|
||||
},
|
||||
v1.EnvVar{
|
||||
Name: "POOL_NAME",
|
||||
Value: poolName,
|
||||
},
|
||||
v1.EnvVar{
|
||||
Name: "ANA_GROUP",
|
||||
Value: anaGroup,
|
||||
},
|
||||
k8sutil.PodIPEnvVar("POD_IP"),
|
||||
)
|
||||
|
||||
privileged := true
|
||||
container := v1.Container{
|
||||
Name: "generate-ceph-conf",
|
||||
Image: cephImage,
|
||||
ImagePullPolicy: imagePullPolicy,
|
||||
Command: []string{
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
script,
|
||||
},
|
||||
Args: cephArgs,
|
||||
Env: envVars,
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
adminKeyringVolumeMount(),
|
||||
cephConfigMount,
|
||||
gatewayConfigMount,
|
||||
},
|
||||
Resources: nvmeof.Spec.Resources,
|
||||
SecurityContext: &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{
|
||||
"SYS_ADMIN",
|
||||
},
|
||||
Drop: []v1.Capability{
|
||||
"NET_RAW",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return container
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) daemonContainer(nvmeof *cephv1.CephNVMeOFGateway, cephConfigMount v1.VolumeMount) v1.Container {
|
||||
privileged := true
|
||||
container := v1.Container{
|
||||
Name: "nvmeof-gateway",
|
||||
Args: []string{
|
||||
"-c",
|
||||
"/etc/ceph/nvmeof.conf",
|
||||
},
|
||||
|
||||
Image: getNVMeOFImage(nvmeof),
|
||||
ImagePullPolicy: v1.PullIfNotPresent,
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
cephConfigMount,
|
||||
},
|
||||
Env: func() []v1.EnvVar {
|
||||
envVars := controller.DaemonEnvVars(r.cephClusterSpec)
|
||||
// Build CEPH_ARGS using DefaultFlags which includes fsid, keyring, logging flags, and mon host flags
|
||||
cephArgs := cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring")
|
||||
cephArgsStr := strings.Join(cephArgs, " ")
|
||||
logger.Infof("CEPH_ARGS for nvmeof gateway: %s", cephArgsStr)
|
||||
envVars = append(envVars, v1.EnvVar{
|
||||
Name: "CEPH_ARGS",
|
||||
Value: cephArgsStr,
|
||||
})
|
||||
return envVars
|
||||
}(),
|
||||
|
||||
Ports: func() []v1.ContainerPort {
|
||||
ioPort, gatewayPort, monitorPort, discoveryPort := getPorts(nvmeof)
|
||||
return []v1.ContainerPort{
|
||||
{
|
||||
Name: "io",
|
||||
ContainerPort: ioPort,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "gateway",
|
||||
ContainerPort: gatewayPort,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "monitor",
|
||||
ContainerPort: monitorPort,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "discovery",
|
||||
ContainerPort: discoveryPort,
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
}
|
||||
}(),
|
||||
|
||||
Resources: nvmeof.Spec.Resources,
|
||||
SecurityContext: &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{
|
||||
"SYS_ADMIN",
|
||||
},
|
||||
Drop: []v1.Capability{
|
||||
"NET_RAW",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if nvmeof.Spec.LivenessProbe != nil && !nvmeof.Spec.LivenessProbe.Disabled && nvmeof.Spec.LivenessProbe.Probe == nil {
|
||||
container.LivenessProbe = r.defaultLivenessProbe(nvmeof)
|
||||
}
|
||||
result := cephconfig.ConfigureLivenessProbe(container, nvmeof.Spec.LivenessProbe)
|
||||
return result
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNVMeOFGateway) defaultLivenessProbe(nvmeof *cephv1.CephNVMeOFGateway) *v1.Probe {
|
||||
ioPort, _, _, _ := getPorts(nvmeof)
|
||||
return controller.GenerateLivenessProbeTcpPort(ioPort, 10)
|
||||
}
|
||||
|
||||
func getLabels(n *cephv1.CephNVMeOFGateway, daemonID string) map[string]string {
|
||||
return controller.CephDaemonAppLabels(
|
||||
AppName, n.Namespace, "nvmeof", n.Name+"-"+daemonID, n.Name,
|
||||
"cephnvmeofgateways.ceph.rook.io", true,
|
||||
)
|
||||
}
|
||||
|
||||
func instanceName(nvmeof *cephv1.CephNVMeOFGateway, daemonID string) string {
|
||||
return fmt.Sprintf("%s-%s-%s", AppName, nvmeof.Name, daemonID)
|
||||
}
|
||||
|
||||
// getNVMeOFImage returns the image to use for the NVMe-oF gateway.
|
||||
// If the image is specified in the spec, it is used; otherwise, the default image is returned.
|
||||
func getNVMeOFImage(nvmeof *cephv1.CephNVMeOFGateway) string {
|
||||
if nvmeof.Spec.Image != "" {
|
||||
return nvmeof.Spec.Image
|
||||
}
|
||||
return defaultNVMeOFImage
|
||||
}
|
||||
|
||||
func gatewayConfigVolumeAndMount(configConfigMap string) (v1.Volume, v1.VolumeMount) {
|
||||
// Mount to /config temporarily - init container will copy to /etc/ceph/nvmeof.conf
|
||||
cfgDir := "/config"
|
||||
cfgVolName := "gateway-config"
|
||||
configMapSource := &v1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: v1.LocalObjectReference{Name: configConfigMap},
|
||||
Items: []v1.KeyToPath{{Key: configKey, Path: "nvmeof.conf"}},
|
||||
}
|
||||
v := v1.Volume{Name: cfgVolName, VolumeSource: v1.VolumeSource{ConfigMap: configMapSource}}
|
||||
m := v1.VolumeMount{Name: cfgVolName, MountPath: cfgDir, ReadOnly: true}
|
||||
return v, m
|
||||
}
|
||||
|
||||
func adminKeyringVolume() v1.Volume {
|
||||
mode := int32(292)
|
||||
return v1.Volume{
|
||||
Name: "ceph-admin-keyring",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
Secret: &v1.SecretVolumeSource{
|
||||
SecretName: "rook-ceph-admin-keyring",
|
||||
Items: []v1.KeyToPath{
|
||||
{
|
||||
Key: "keyring",
|
||||
Path: "keyring",
|
||||
Mode: &mode,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func adminKeyringVolumeMount() v1.VolumeMount {
|
||||
return v1.VolumeMount{
|
||||
Name: "ceph-admin-keyring",
|
||||
MountPath: "/tmp/ceph/",
|
||||
ReadOnly: true,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
/*
|
||||
Copyright 2025 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package nvmeof
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
||||
cephTest "github.com/rook/rook/pkg/operator/ceph/test"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
optest "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNVMeOFGateway, string) {
|
||||
clientset := optest.New(t, 1)
|
||||
c := &clusterd.Context{
|
||||
Executor: &exectest.MockExecutor{},
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
object := []runtime.Object{
|
||||
&cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
r := &ReconcileCephNVMeOFGateway{
|
||||
client: cl,
|
||||
scheme: scheme.Scheme,
|
||||
context: c,
|
||||
clusterInfo: &cephclient.ClusterInfo{
|
||||
FSID: "myfsid",
|
||||
CephVersion: cephver.Squid,
|
||||
},
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{
|
||||
Image: "quay.io/ceph/ceph:v19",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
configHash := "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101"
|
||||
|
||||
return r, configHash
|
||||
}
|
||||
|
||||
func TestDeploymentSpec(t *testing.T) {
|
||||
t.Run("basic", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
PriorityClassName: "my-priority-class",
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
assert.Equal(t, configHash, d.Spec.Template.Annotations["config-hash"])
|
||||
|
||||
// Verify deployment labels contain Ceph requirements
|
||||
daemonID := nvmeof.Name + "-0"
|
||||
cephTest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels,
|
||||
"nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name,
|
||||
"cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof")
|
||||
|
||||
podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template)
|
||||
podTemplate.RunFullSuite(
|
||||
AppName,
|
||||
optest.ResourceLimitExpectations{
|
||||
CPUResourceLimit: "500",
|
||||
MemoryResourceLimit: "1Ki",
|
||||
CPUResourceRequest: "200",
|
||||
MemoryResourceRequest: "512",
|
||||
},
|
||||
)
|
||||
assert.Equal(t, "my-priority-class", d.Spec.Template.Spec.PriorityClassName)
|
||||
assert.Equal(t, serviceAccountName, d.Spec.Template.Spec.ServiceAccountName)
|
||||
})
|
||||
|
||||
t.Run("basic with default liveness-probe", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
PriorityClassName: "my-priority-class",
|
||||
LivenessProbe: &cephv1.ProbeSpec{
|
||||
Disabled: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
|
||||
var nvmeofCont *v1.Container = nil
|
||||
for i := range d.Spec.Template.Spec.Containers {
|
||||
if d.Spec.Template.Spec.Containers[i].Name == "nvmeof-gateway" {
|
||||
nvmeofCont = &d.Spec.Template.Spec.Containers[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotNil(t, nvmeofCont)
|
||||
assert.NotNil(t, nvmeofCont.LivenessProbe)
|
||||
assert.Equal(t, nvmeofCont.LivenessProbe.InitialDelaySeconds, int32(10))
|
||||
assert.Equal(t, nvmeofCont.LivenessProbe.FailureThreshold, int32(10))
|
||||
assert.GreaterOrEqual(t, nvmeofCont.LivenessProbe.TimeoutSeconds, int32(5))
|
||||
})
|
||||
|
||||
t.Run("with host network", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
r.cephClusterSpec.Network = cephv1.NetworkSpec{
|
||||
HostNetwork: true,
|
||||
}
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, d.Spec.Template.Spec.HostNetwork)
|
||||
assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy)
|
||||
|
||||
svc := r.generateCephNVMeOFService(nvmeof, "0")
|
||||
assert.Equal(t, v1.ClusterIPNone, svc.Spec.ClusterIP)
|
||||
})
|
||||
|
||||
t.Run("with custom configmap ref", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
ConfigMapRef: "custom-configmap",
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := "custom-configmap"
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
|
||||
var gatewayConfigVol *v1.Volume = nil
|
||||
for i := range d.Spec.Template.Spec.Volumes {
|
||||
if d.Spec.Template.Spec.Volumes[i].Name == "gateway-config" {
|
||||
gatewayConfigVol = &d.Spec.Template.Spec.Volumes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.NotNil(t, gatewayConfigVol)
|
||||
assert.NotNil(t, gatewayConfigVol.ConfigMap)
|
||||
assert.Equal(t, "custom-configmap", gatewayConfigVol.ConfigMap.Name)
|
||||
})
|
||||
|
||||
t.Run("init container configuration", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Len(t, d.Spec.Template.Spec.InitContainers, 1)
|
||||
initCont := d.Spec.Template.Spec.InitContainers[0]
|
||||
assert.Equal(t, "generate-ceph-conf", initCont.Name)
|
||||
assert.Equal(t, r.cephClusterSpec.CephVersion.Image, initCont.Image)
|
||||
assert.Equal(t, []string{"/bin/bash", "-c", connectionConfigScript}, initCont.Command)
|
||||
assert.Contains(t, initCont.Args, "--keyring=/etc/ceph/keyring")
|
||||
|
||||
assertEnvVar(t, initCont.Env, "GATEWAY_NAME", instanceName(nvmeof, "0"))
|
||||
assertEnvVar(t, initCont.Env, "POOL_NAME", nvmeof.Spec.Pool)
|
||||
assertEnvVar(t, initCont.Env, "ANA_GROUP", nvmeof.Spec.Group)
|
||||
assertEnvVarPresent(t, initCont.Env, "POD_IP")
|
||||
|
||||
volumeMountNames := []string{}
|
||||
for _, vm := range initCont.VolumeMounts {
|
||||
volumeMountNames = append(volumeMountNames, vm.Name)
|
||||
}
|
||||
assert.Contains(t, volumeMountNames, "ceph-admin-keyring")
|
||||
assert.Contains(t, volumeMountNames, "gateway-config")
|
||||
assert.Contains(t, volumeMountNames, "ceph-conf-emptydir")
|
||||
})
|
||||
|
||||
t.Run("daemon container configuration", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Len(t, d.Spec.Template.Spec.Containers, 1)
|
||||
daemonCont := d.Spec.Template.Spec.Containers[0]
|
||||
assert.Equal(t, "nvmeof-gateway", daemonCont.Name)
|
||||
assert.Equal(t, nvmeof.Spec.Image, daemonCont.Image)
|
||||
|
||||
expectedCephArgs := strings.Join(cephconfig.DefaultFlags(r.clusterInfo.FSID, "/etc/ceph/keyring"), " ")
|
||||
assertEnvVar(t, daemonCont.Env, "CEPH_ARGS", expectedCephArgs)
|
||||
|
||||
portNames := []string{}
|
||||
for _, p := range daemonCont.Ports {
|
||||
portNames = append(portNames, p.Name)
|
||||
}
|
||||
assert.Contains(t, portNames, "io")
|
||||
assert.Contains(t, portNames, "gateway")
|
||||
assert.Contains(t, portNames, "monitor")
|
||||
assert.Contains(t, portNames, "discovery")
|
||||
|
||||
assert.NotNil(t, daemonCont.SecurityContext)
|
||||
// SecurityContext should be set with privileged=true for NVMeOF gateway
|
||||
assert.NotNil(t, daemonCont.SecurityContext.Privileged)
|
||||
assert.True(t, *daemonCont.SecurityContext.Privileged)
|
||||
})
|
||||
|
||||
t.Run("daemon container uses default image when not specified", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
// Image not specified - should use default
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Len(t, d.Spec.Template.Spec.Containers, 1)
|
||||
daemonCont := d.Spec.Template.Spec.Containers[0]
|
||||
assert.Equal(t, "nvmeof-gateway", daemonCont.Name)
|
||||
assert.Equal(t, "quay.io/ceph/nvmeof:1.5", daemonCont.Image)
|
||||
})
|
||||
|
||||
t.Run("hostname set when instance name is valid", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, instanceName(nvmeof, "0"), d.Spec.Template.Spec.Hostname)
|
||||
})
|
||||
|
||||
t.Run("hostname not set when instance name is invalid", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my_nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, d.Spec.Template.Spec.Hostname)
|
||||
})
|
||||
|
||||
t.Run("service generation", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
},
|
||||
}
|
||||
|
||||
r, _ := newDeploymentSpecTest(t)
|
||||
|
||||
svc := r.generateCephNVMeOFService(nvmeof, "0")
|
||||
assert.Equal(t, "rook-ceph-nvmeof-my-nvmeof-0", svc.Name)
|
||||
assert.Equal(t, "rook-ceph-test-ns", svc.Namespace)
|
||||
|
||||
// Verify service labels contain Ceph requirements
|
||||
daemonID := nvmeof.Name + "-0"
|
||||
cephTest.AssertLabelsContainCephRequirements(t, svc.Labels,
|
||||
"nvmeof", daemonID, AppName, nvmeof.Namespace, nvmeof.Name,
|
||||
"cephnvmeofgateways.ceph.rook.io", "ceph-nvmeof")
|
||||
|
||||
portNames := []string{}
|
||||
for _, p := range svc.Spec.Ports {
|
||||
portNames = append(portNames, p.Name)
|
||||
}
|
||||
assert.Contains(t, portNames, "io")
|
||||
assert.Contains(t, portNames, "gateway")
|
||||
assert.Contains(t, portNames, "monitor")
|
||||
assert.Contains(t, portNames, "discovery")
|
||||
|
||||
for _, p := range svc.Spec.Ports {
|
||||
switch p.Name {
|
||||
case "io":
|
||||
assert.Equal(t, int32(nvmeofIOPort), p.Port)
|
||||
case "gateway":
|
||||
assert.Equal(t, int32(nvmeofGatewayPort), p.Port)
|
||||
case "monitor":
|
||||
assert.Equal(t, int32(nvmeofMonitorPort), p.Port)
|
||||
case "discovery":
|
||||
assert.Equal(t, int32(nvmeofDiscoveryPort), p.Port)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("custom ports", func(t *testing.T) {
|
||||
nvmeof := &cephv1.CephNVMeOFGateway{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nvmeof",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NVMeOFGatewaySpec{
|
||||
Image: "quay.io/ceph/nvmeof:1.5",
|
||||
Pool: "nvmeof",
|
||||
Group: "group-a",
|
||||
Instances: 1,
|
||||
Ports: &cephv1.NVMeOFGatewayPorts{
|
||||
IOPort: 14420,
|
||||
GatewayPort: 15500,
|
||||
MonitorPort: 15499,
|
||||
DiscoveryPort: 18009,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r, configHash := newDeploymentSpecTest(t)
|
||||
|
||||
configMapName := fmt.Sprintf("rook-ceph-nvmeof-%s-config", nvmeof.Name)
|
||||
d, err := r.makeDeployment(nvmeof, "0", configMapName, configHash)
|
||||
assert.NoError(t, err)
|
||||
|
||||
daemonCont := d.Spec.Template.Spec.Containers[0]
|
||||
assertContainerPort(t, daemonCont.Ports, "io", 14420)
|
||||
assertContainerPort(t, daemonCont.Ports, "gateway", 15500)
|
||||
assertContainerPort(t, daemonCont.Ports, "monitor", 15499)
|
||||
assertContainerPort(t, daemonCont.Ports, "discovery", 18009)
|
||||
|
||||
svc := r.generateCephNVMeOFService(nvmeof, "0")
|
||||
assertServicePort(t, svc.Spec.Ports, "io", 14420)
|
||||
assertServicePort(t, svc.Spec.Ports, "gateway", 15500)
|
||||
assertServicePort(t, svc.Spec.Ports, "monitor", 15499)
|
||||
assertServicePort(t, svc.Spec.Ports, "discovery", 18009)
|
||||
})
|
||||
}
|
||||
|
||||
func assertEnvVar(t *testing.T, envVars []v1.EnvVar, name, value string) {
|
||||
t.Helper()
|
||||
for _, env := range envVars {
|
||||
if env.Name == name {
|
||||
assert.Equal(t, value, env.Value)
|
||||
return
|
||||
}
|
||||
}
|
||||
assert.Failf(t, "missing env var", "expected %s", name)
|
||||
}
|
||||
|
||||
func assertEnvVarPresent(t *testing.T, envVars []v1.EnvVar, name string) {
|
||||
t.Helper()
|
||||
for _, env := range envVars {
|
||||
if env.Name == name {
|
||||
assert.NotNil(t, env.ValueFrom)
|
||||
return
|
||||
}
|
||||
}
|
||||
assert.Failf(t, "missing env var", "expected %s", name)
|
||||
}
|
||||
|
||||
func assertServicePort(t *testing.T, ports []v1.ServicePort, name string, value int32) {
|
||||
t.Helper()
|
||||
for _, port := range ports {
|
||||
if port.Name == name {
|
||||
assert.Equal(t, value, port.Port)
|
||||
return
|
||||
}
|
||||
}
|
||||
assert.Failf(t, "missing port", "expected %s", name)
|
||||
}
|
||||
|
||||
func assertContainerPort(t *testing.T, ports []v1.ContainerPort, name string, value int32) {
|
||||
t.Helper()
|
||||
for _, port := range ports {
|
||||
if port.Name == name {
|
||||
assert.Equal(t, value, port.ContainerPort)
|
||||
return
|
||||
}
|
||||
}
|
||||
assert.Failf(t, "missing port", "expected %s", name)
|
||||
}
|
||||
Reference in new issue
Block a user