Merge pull request #17243 from rook/mergify/bp/release-1.19/pr-17154
csi: use ceph-csi-operator to deploy Ceph-CSI/NVMe-oF (backport #17154)
This commit is contained in:
10 files changed
+1107
-2499
No files matched your search
@@ -1995,13 +1995,11 @@ jobs:
|
||||
tests/scripts/github-action-helper.sh use_local_disk
|
||||
tests/scripts/github-action-helper.sh create_partitions_for_osds
|
||||
|
||||
- name: deploy cluster without ceph-csi operator
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
OPERATOR_MANIFEST="deploy/examples/operator.yaml"
|
||||
sed -i 's|# ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.16.2"|ROOK_CSI_CEPH_IMAGE: "quay.io/nixpanic/cephcsi:nvmeof"|' "${OPERATOR_MANIFEST}"
|
||||
sed -i 's|ROOK_USE_CSI_OPERATOR: "true"|ROOK_USE_CSI_OPERATOR: "false"|' "${OPERATOR_MANIFEST}"
|
||||
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "${OPERATOR_MANIFEST}"
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/operator.yaml"
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi-operator.yaml"
|
||||
tests/scripts/github-action-helper.sh deploy_manifest_with_local_build "deploy/examples/csi/nvmeof/driver.yaml"
|
||||
|
||||
export BLOCK="$(tests/scripts/github-action-helper.sh find_extra_block_dev)"
|
||||
yq w -i -d0 deploy/examples/cluster-test.yaml spec.storage.deviceFilter "${BLOCK}"
|
||||
|
||||
@@ -27,7 +27,6 @@ This guide assumes a Rook cluster as explained in the [Quickstart Guide](../../G
|
||||
### Requirements
|
||||
|
||||
- **Ceph Version**: Ceph v20 (Tentacle) or later
|
||||
- **Disable the Ceph CSI operator**: We are still updating the Ceph CSI operator with NVMe-oF support. Currently, it is required to disable the CSI operator to test NVMe-oF. In operator.yaml, set `ROOK_USE_CSI_OPERATOR: "false"`.
|
||||
|
||||
## Step 1: Create a Ceph Block Pool
|
||||
|
||||
@@ -97,81 +96,33 @@ NAME READY STATUS RESTARTS AGE
|
||||
rook-ceph-nvmeof-nvmeof-a-85844ff6b8-4r8gj 1/1 Running 0 91s
|
||||
```
|
||||
|
||||
## Step 4: Deploy the NVMe-oF CSI Driver
|
||||
## Step 3: Deploy the NVMe-oF CSI Driver via CSI Operator
|
||||
|
||||
The NVMe-oF CSI driver handles dynamic provisioning of volumes. Deploy the CSI provisioner with the NVMe-oF driver.
|
||||
The NVMe-oF CSI driver is deployed via the ceph-csi operator.
|
||||
|
||||
Deploy the NVMe-oF CSI provisioner from the example manifest:
|
||||
Apply the `Driver` CR for NVMe-oF that will trigger the creation of the
|
||||
Ceph-CSI/NVMe-oF deployment and daemonset:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/provisioner.yaml
|
||||
kubectl create -f deploy/examples/csi/nvmeof/driver.yaml
|
||||
```
|
||||
|
||||
Verify the CSI provisioner pod is running:
|
||||
Verify the CSI operator created the controller and node plugins:
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=csi-nvmeofplugin-provisioner
|
||||
kubectl get pods -n rook-ceph | grep nvmeof
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
csi-nvmeofplugin-provisioner-65b4fbbc8-jjsqj 4/4 Running 0 75s
|
||||
rook-ceph.nvmeof.csi.ceph.com-ctrlplugin-d9d77fb7c-kkl28 5/5 Running 0 60s
|
||||
rook-ceph.nvmeof.csi.ceph.com-nodeplugin-xvt5g 2/2 Running 0 60s
|
||||
```
|
||||
|
||||
## Step 5: Create the StorageClass
|
||||
|
||||
Create a StorageClass that uses the NVMe-oF CSI driver. You'll need to gather the following information from the gateway:
|
||||
|
||||
1. **nvmeofGatewayAddress**: A stable address for the gateway management API
|
||||
2. **nvmeofGatewayPort**: The gateway port (default: 5500)
|
||||
3. **listeners**: A JSON array containing listener information for each gateway instance
|
||||
|
||||
Discover the values to use in the StorageClass:
|
||||
|
||||
1. **nvmeofGatewayAddress**: Use the Service `CLUSTER-IP`.
|
||||
|
||||
```console
|
||||
kubectl get service -n rook-ceph rook-ceph-nvmeof-nvmeof-a
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
rook-ceph-nvmeof-nvmeof-a ClusterIP 10.106.98.71 <none> 4420/TCP,5500/TCP,5499/TCP,8009/TCP 24m
|
||||
```
|
||||
|
||||
2. **listeners.address**: Use the gateway pod IP.
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=rook-ceph-nvmeof -o wide
|
||||
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
|
||||
rook-ceph-nvmeof-nvmeof-a-5fd6cd4d46-mrbwk 1/1 Running 0 26m 10.244.0.16 minikube <none> <none>
|
||||
```
|
||||
|
||||
3. **listeners.hostname**: Use the gateway deployment name.
|
||||
|
||||
```console
|
||||
kubectl get deployments.apps -n rook-ceph -l app=rook-ceph-nvmeof
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY UP-TO-DATE AVAILABLE AGE
|
||||
rook-ceph-nvmeof-nvmeof-a 1/1 1 1 27m
|
||||
```
|
||||
|
||||
Create the StorageClass:
|
||||
## Step 4: Create the StorageClass
|
||||
|
||||
Create a StorageClass that uses the NVMe-oF CSI driver.
|
||||
|
||||
```yaml
|
||||
apiVersion: storage.k8s.io/v1
|
||||
@@ -182,16 +133,11 @@ parameters:
|
||||
clusterID: rook-ceph
|
||||
pool: nvmeof
|
||||
subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
|
||||
# Management API - talks to gateway to create subsystems/namespaces
|
||||
nvmeofGatewayAddress: "10.106.98.71"
|
||||
nvmeofGatewayAddress: "rook-ceph-nvmeof-nvmeof-a.rook-ceph.svc.cluster.local"
|
||||
nvmeofGatewayPort: "5500"
|
||||
# Data Plane - worker nodes connect here for actual I/O
|
||||
# List ALL gateway pods for HA and multipath
|
||||
listeners: |
|
||||
[
|
||||
{
|
||||
"address": "10.244.0.16",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-a"
|
||||
}
|
||||
]
|
||||
@@ -201,23 +147,27 @@ parameters:
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/controller-modify-secret-name: rook-csi-rbd-provisioner
|
||||
csi.storage.k8s.io/controller-modify-secret-namespace: rook-ceph
|
||||
csi.storage.k8s.io/node-expand-secret-name: rook-csi-rbd-node
|
||||
csi.storage.k8s.io/node-expand-secret-namespace: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
|
||||
provisioner: nvmeof.csi.ceph.com
|
||||
provisioner: rook-ceph.nvmeof.csi.ceph.com
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
allowVolumeExpansion: true
|
||||
```
|
||||
|
||||
Create the StorageClass:
|
||||
!!! note
|
||||
The provisioner name `rook-ceph.nvmeof.csi.ceph.com` is prefixed
|
||||
with the operator namespace.
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/storageclass.yaml
|
||||
```
|
||||
|
||||
## Step 6: Create a PersistentVolumeClaim
|
||||
## Step 5: Create a PersistentVolumeClaim
|
||||
|
||||
Create a PVC using the NVMe-oF storage class:
|
||||
|
||||
@@ -258,36 +208,9 @@ NAME STATUS VOLUME CAP
|
||||
nvmeof-external-volume Bound pvc-b4108580-5cfa-46d3-beff-320088a5bf3c 128Mi RWO ceph-nvmeof 20m
|
||||
```
|
||||
|
||||
## Step 7: Deploy the NVMe-oF CSI Node Plugin
|
||||
## Step 6: Create a Pod
|
||||
|
||||
Deploy the NVMe-oF CSI node plugin:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/node-plugin.yaml
|
||||
```
|
||||
|
||||
Verify the node plugin pod is running:
|
||||
|
||||
```console
|
||||
kubectl get pods -n rook-ceph -l app=nvmeof.csi.ceph.com-nodeplugin
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
nvmeof.csi.ceph.com-nodeplugin-xnm82 2/2 Running 0 31h
|
||||
```
|
||||
|
||||
## Step 8: Accessing Volumes via NVMe-oF
|
||||
|
||||
Once the PVC is created and bound, the volume is available via NVMe-oF. The volume can be accessed by both Kubernetes pods within the cluster and external clients outside the cluster.
|
||||
|
||||
### Access from Kubernetes Pods
|
||||
|
||||
Kubernetes pods can consume NVMe-oF volumes by mounting the PVC directly. The CSI driver handles the NVMe-oF connection automatically when the pod mounts the volume.
|
||||
|
||||
Create a sample pod that mounts the PVC:
|
||||
Create a pod that consumes the NVMe-oF volume:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/csi/nvmeof/pod.yaml
|
||||
@@ -296,16 +219,22 @@ kubectl create -f deploy/examples/csi/nvmeof/pod.yaml
|
||||
Verify the pod is running:
|
||||
|
||||
```console
|
||||
kubectl get pods -n default
|
||||
kubectl get pods -n default nvmeof-test-pod
|
||||
```
|
||||
|
||||
**Example Output**
|
||||
|
||||
```console
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
nvmeof-test-pod 1/1 Running 0 29h
|
||||
nvmeof-test-pod 1/1 Running 0 60s
|
||||
```
|
||||
|
||||
## Step 7: Accessing Volumes via NVMe-oF
|
||||
|
||||
Once the PVC is created and bound, the volume is available via
|
||||
NVMe-oF. The volume can be accessed by both Kubernetes pods within
|
||||
the cluster and external clients outside the cluster.
|
||||
|
||||
### Access from External Clients
|
||||
|
||||
External clients outside the Kubernetes cluster can connect to the gateway using standard NVMe-oF procedures.
|
||||
@@ -366,7 +295,7 @@ sudo mount /dev/nvmeXnY /mnt/nvmeof
|
||||
For production deployments, configure multiple gateway instances for high availability:
|
||||
|
||||
1. **Increase Gateway Instances**: Set `instances: 2` or higher in the `CephNVMeOFGateway` spec
|
||||
2. **Update StorageClass Listeners**: Add all gateway instance addresses and instance names to the `listeners` array
|
||||
2. **Update StorageClass Listeners**: Add all gateway deployment hostnames to the `listeners` array
|
||||
3. **Load Balancing**: Each gateway instance has its own Service; list all of them to support multipath/HA
|
||||
|
||||
Example with multiple instances:
|
||||
@@ -377,19 +306,16 @@ spec:
|
||||
# ... other settings
|
||||
```
|
||||
|
||||
Then update the StorageClass `listeners` to include all gateway instances/services:
|
||||
Then update the StorageClass `listeners` to include all gateway
|
||||
hostnames:
|
||||
|
||||
```yaml
|
||||
listeners: |
|
||||
[
|
||||
{
|
||||
"address": "10.99.212.218",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-a"
|
||||
},
|
||||
{
|
||||
"address": "10.99.212.219",
|
||||
"port": 4420,
|
||||
"hostname": "rook-ceph-nvmeof-nvmeof-b"
|
||||
}
|
||||
]
|
||||
@@ -403,16 +329,16 @@ listeners: |
|
||||
kubectl logs -n rook-ceph -l app=rook-ceph-nvmeof --tail=100
|
||||
```
|
||||
|
||||
### Check CSI Provisioner Logs
|
||||
### Check CSI Controller Plugin Logs
|
||||
|
||||
```console
|
||||
kubectl logs -n rook-ceph -l app=csi-nvmeofplugin-provisioner --tail=100
|
||||
kubectl logs -n rook-ceph deploy/rook-ceph.nvmeof.csi.ceph.com-ctrlplugin --tail=100
|
||||
```
|
||||
|
||||
### Verify Gateway Service
|
||||
|
||||
```console
|
||||
kubectl describe service -n rook-ceph rook-ceph-nvmeof-my-nvmeof-0
|
||||
kubectl describe service -n rook-ceph rook-ceph-nvmeof-nvmeof-a
|
||||
```
|
||||
|
||||
### Check PVC Events
|
||||
@@ -446,17 +372,14 @@ kubectl delete pvc nvmeof-external-volume
|
||||
# Delete the StorageClass
|
||||
kubectl delete storageclass ceph-nvmeof
|
||||
|
||||
# Delete the NVMe-oF CSI node plugin
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/node-plugin.yaml
|
||||
|
||||
# Delete the NVMe-oF CSI provisioner (includes ServiceAccount/RBAC)
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/provisioner.yaml
|
||||
# Delete the NVMe-oF CSI operator resources
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/csi-operator-nvmeof.yaml
|
||||
|
||||
# Delete the NVMe-oF gateway
|
||||
kubectl delete -n rook-ceph cephnvmeofgateway.ceph.rook.io my-nvmeof
|
||||
kubectl delete -f deploy/examples/nvmeof-test.yaml
|
||||
|
||||
# Delete the block pool (optional)
|
||||
kubectl delete -n rook-ceph cephblockpool.ceph.rook.io nvmeof
|
||||
kubectl delete -f deploy/examples/csi/nvmeof/nvmeof-pool.yaml
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
+1021
-1794
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,8 @@
|
||||
apiVersion: csi.ceph.io/v1
|
||||
kind: Driver
|
||||
metadata:
|
||||
name: rook-ceph.nvmeof.csi.ceph.com
|
||||
namespace: rook-ceph # namespace:operator
|
||||
spec:
|
||||
controllerPlugin:
|
||||
hostNetwork: false
|
||||
@@ -1,202 +0,0 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: nvmeof.csi.ceph.com-nodeplugin
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
app: nvmeof.csi.ceph.com-nodeplugin
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: nvmeof.csi.ceph.com-nodeplugin
|
||||
spec:
|
||||
containers:
|
||||
- args:
|
||||
- --v=5
|
||||
- --type=nvmeof
|
||||
- --nodeserver=true
|
||||
- --nodeid=$(NODE_ID)
|
||||
- --drivername=nvmeof.csi.ceph.com
|
||||
- --endpoint=unix:///csi/csi.sock
|
||||
- --pidlimit=-1
|
||||
- --csi-addons-endpoint=unix:///csi/csi-addons.sock
|
||||
- --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/
|
||||
- --logtostderr=false
|
||||
- --alsologtostderr=true
|
||||
- --log_file=csi-logs/csi-nvmeofplugin.log
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: status.podIP
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
image: quay.io/cephcsi/cephcsi:v3.16.2
|
||||
imagePullPolicy: Always
|
||||
name: csi-nvmeofplugin
|
||||
resources: {}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: true
|
||||
capabilities:
|
||||
add:
|
||||
- SYS_ADMIN
|
||||
drop:
|
||||
- All
|
||||
privileged: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /run/mount
|
||||
name: host-run-mount
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
- mountPath: /tmp/csi/keys
|
||||
name: keys-tmp-dir
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
# FIXED: Mount path should end with / to mount as directory
|
||||
- mountPath: /etc/ceph-csi-config/
|
||||
name: ceph-csi-config
|
||||
- mountPath: /var/lib/kubelet/plugins
|
||||
mountPropagation: Bidirectional
|
||||
name: plugin-mount-dir
|
||||
- mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: Bidirectional
|
||||
name: pods-mount-dir
|
||||
- mountPath: /etc/selinux
|
||||
name: etc-selinux
|
||||
readOnly: true
|
||||
- mountPath: /run/secrets/tokens
|
||||
name: oidc-token
|
||||
readOnly: true
|
||||
- mountPath: /csi-logs
|
||||
name: logs-dir
|
||||
# Log rotation mount - commented out since ConfigMap is optional
|
||||
# - mountPath: /etc/logrotate.d
|
||||
# name: log-rotate-dir
|
||||
# readOnly: true
|
||||
- args:
|
||||
- "--v=1"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock"
|
||||
image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: driver-registrar
|
||||
resources: {}
|
||||
securityContext:
|
||||
capabilities:
|
||||
drop:
|
||||
- All
|
||||
privileged: true
|
||||
terminationMessagePath: /dev/termination-log
|
||||
terminationMessagePolicy: File
|
||||
volumeMounts:
|
||||
- mountPath: /csi
|
||||
name: plugin-dir
|
||||
- mountPath: /registration
|
||||
name: registration-dir
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
priorityClassName: system-node-critical
|
||||
restartPolicy: Always
|
||||
schedulerName: default-scheduler
|
||||
securityContext: {}
|
||||
# FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend
|
||||
# This service account already exists and has the necessary permissions
|
||||
serviceAccount: rook-csi-rbd-plugin-sa
|
||||
serviceAccountName: rook-csi-rbd-plugin-sa
|
||||
terminationGracePeriodSeconds: 30
|
||||
# tolerations:
|
||||
# - effect: NoSchedule
|
||||
# key: node.ocs.openshift.io/storage
|
||||
# operator: Equal
|
||||
# value: "true"
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /dev
|
||||
type: ""
|
||||
name: host-dev
|
||||
- hostPath:
|
||||
path: /sys
|
||||
type: ""
|
||||
name: host-sys
|
||||
- hostPath:
|
||||
path: /run/mount
|
||||
type: ""
|
||||
name: host-run-mount
|
||||
- hostPath:
|
||||
path: /lib/modules
|
||||
type: ""
|
||||
name: lib-modules
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
name: keys-tmp-dir
|
||||
# FIXED: Use rook-ceph-csi-config instead of ceph-csi-config
|
||||
# This ConfigMap contains the cluster configuration with monitor endpoints
|
||||
- configMap:
|
||||
name: rook-ceph-csi-config
|
||||
items:
|
||||
- key: csi-cluster-config-json
|
||||
path: config.json
|
||||
name: ceph-csi-config
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
name: plugin-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins
|
||||
type: Directory
|
||||
name: plugin-mount-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
name: pods-mount-dir
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins_registry
|
||||
type: Directory
|
||||
name: registration-dir
|
||||
- hostPath:
|
||||
path: /etc/selinux
|
||||
type: ""
|
||||
name: etc-selinux
|
||||
- name: oidc-token
|
||||
projected:
|
||||
defaultMode: 420
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
audience: ceph-csi-kms
|
||||
expirationSeconds: 3600
|
||||
path: oidc-token
|
||||
- hostPath:
|
||||
path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin
|
||||
type: DirectoryOrCreate
|
||||
name: logs-dir
|
||||
# Log rotation ConfigMap - optional, can be created if log rotation is needed
|
||||
# - configMap:
|
||||
# defaultMode: 420
|
||||
# name: nvmeof.csi.ceph.com-logrotate-config
|
||||
# name: log-rotate-dir
|
||||
updateStrategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
type: RollingUpdate
|
||||
@@ -1,339 +0,0 @@
|
||||
---
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: ["events.k8s.io"]
|
||||
resources: ["events"]
|
||||
verbs: ["create", "patch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update", "patch", "create"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots/status"]
|
||||
verbs: ["get", "list", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments/status"]
|
||||
verbs: ["patch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["csinodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["serviceaccounts"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["serviceaccounts/token"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotcontents"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: ["groupsnapshot.storage.k8s.io"]
|
||||
resources: ["volumegroupsnapshotcontents/status"]
|
||||
verbs: ["update", "patch"]
|
||||
- apiGroups: ["replication.storage.openshift.io"]
|
||||
resources: ["volumegroupreplicationcontents"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["replication.storage.openshift.io"]
|
||||
resources: ["volumegroupreplicationclasses"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattributesclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims/status"]
|
||||
verbs: ["patch"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: nvmeof-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
name: nvmeof-external-provisioner-cfg
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "delete"]
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: nvmeof-csi-provisioner-role-cfg
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: nvmeof-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: nvmeof-external-provisioner-cfg
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
# ---
|
||||
# # ssc based on the Ceph-CSI ssc
|
||||
# # the "users:" contains a reference to the "rook-ceph" namespace
|
||||
# #
|
||||
# kind: SecurityContextConstraints
|
||||
# apiVersion: security.openshift.io/v1
|
||||
# metadata:
|
||||
# name: "ceph-nvmeof-provisioner"
|
||||
# # To allow running privilegedContainers
|
||||
# allowPrivilegedContainer: true
|
||||
# # SYS_ADMIN is needed for rbd to execute rbd map command
|
||||
# allowedCapabilities:
|
||||
# - "SYS_ADMIN"
|
||||
# # Set to false as we write to RootFilesystem inside container
|
||||
# readOnlyRootFilesystem: false
|
||||
# requiredDropCapabilities:
|
||||
# - ALL
|
||||
# # The type of volumes which are mounted to csi pods
|
||||
# volumes:
|
||||
# - configMap
|
||||
# - projected
|
||||
# - emptyDir
|
||||
# runAsUser:
|
||||
# type: RunAsAny
|
||||
# seLinuxContext:
|
||||
# type: RunAsAny
|
||||
# users:
|
||||
# # A user needs to be added for each service account.
|
||||
# - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner"
|
||||
---
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: csi-nvmeofplugin-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-nvmeofplugin-provisioner
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
openshift.io/scc: rook-ceph-csi
|
||||
labels:
|
||||
app: csi-nvmeofplugin-provisioner
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: app
|
||||
operator: In
|
||||
values:
|
||||
- csi-nvmeofplugin-provisioner
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
serviceAccountName: nvmeof-csi-provisioner
|
||||
priorityClassName: system-cluster-critical
|
||||
containers:
|
||||
- name: csi-nvmeofplugin
|
||||
# for stable functionality replace canary with latest release version
|
||||
# image: quay.io/gdidi/cephcsi:nvmeof
|
||||
image: quay.io/cephcsi/cephcsi:v3.16.2
|
||||
command: ["/usr/local/bin/cephcsi"]
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--type=nvmeof"
|
||||
- "--controllerserver=true"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=nvmeof.csi.ceph.com"
|
||||
- "--pidlimit=-1"
|
||||
- "--rbdhardmaxclonedepth=8"
|
||||
- "--rbdsoftmaxclonedepth=4"
|
||||
- "--enableprofiling=false"
|
||||
- "--setmetadata=true"
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: ceph-csi-config
|
||||
mountPath: /etc/ceph-csi-config/
|
||||
- name: keys-tmp-dir
|
||||
mountPath: /tmp/csi/keys
|
||||
- name: oidc-token
|
||||
mountPath: /run/secrets/tokens
|
||||
readOnly: true
|
||||
- name: csi-provisioner
|
||||
# image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0
|
||||
image: quay.io/nixpanic/csi-provisioner:pr1440
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=1"
|
||||
- "--timeout=150s"
|
||||
- "--retry-interval-start=500ms"
|
||||
- "--leader-election=true"
|
||||
- "--feature-gates=HonorPVReclaimPolicy=true"
|
||||
- "--prevent-volume-mode-conversion=true"
|
||||
# if fstype is not specified in storageclass, ext4 is default
|
||||
- "--default-fstype=ext4"
|
||||
- "--extra-create-metadata=true"
|
||||
- "--immediate-topology=false"
|
||||
- "--http-endpoint=$(POD_IP):8090"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
ports:
|
||||
- containerPort: 8090
|
||||
name: provisioner
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: csi-attacher
|
||||
image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0
|
||||
args:
|
||||
- "--v=1"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--leader-election=true"
|
||||
- "--retry-interval-start=500ms"
|
||||
- "--default-fstype=ext4"
|
||||
- "--http-endpoint=$(POD_IP):8093"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi-provisioner.sock
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
ports:
|
||||
- containerPort: 8093
|
||||
name: attacher
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: csi-resizer
|
||||
image: quay.io/nixpanic/csi-resizer:pr544
|
||||
# image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
- "--leader-election=true"
|
||||
- "--handle-volume-inuse-error=false"
|
||||
- "--feature-gates=VolumeAttributesClass=true"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
emptyDir:
|
||||
medium: "Memory"
|
||||
# FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config
|
||||
# This ConfigMap contains the cluster configuration with monitor endpoints
|
||||
# that Ceph-CSI needs to connect to the Ceph cluster
|
||||
- name: ceph-csi-config
|
||||
configMap:
|
||||
name: rook-ceph-csi-config
|
||||
items:
|
||||
- key: csi-cluster-config-json
|
||||
path: config.json
|
||||
- name: keys-tmp-dir
|
||||
emptyDir:
|
||||
medium: "Memory"
|
||||
# Only if nvmeof support encryption
|
||||
- name: oidc-token
|
||||
projected:
|
||||
sources:
|
||||
- serviceAccountToken:
|
||||
path: oidc-token
|
||||
expirationSeconds: 3600
|
||||
audience: ceph-csi-kms
|
||||
@@ -7,7 +7,7 @@ parameters:
|
||||
pool: nvmeof
|
||||
subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
|
||||
# Management API - talks to gateway to create subsystems/namespaces
|
||||
nvmeofGatewayAddress: "172.30.47.120"
|
||||
nvmeofGatewayAddress: "rook-ceph-nvmeof-nvmeof-a.rook-ceph.svc.cluster.local"
|
||||
nvmeofGatewayPort: "5500"
|
||||
# Data Plane - worker nodes connect here for actual I/O
|
||||
# List ALL gateway pods for HA and multipath
|
||||
@@ -27,7 +27,7 @@ parameters:
|
||||
csi.storage.k8s.io/node-expand-secret-namespace: rook-ceph
|
||||
imageFormat: "2"
|
||||
imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
|
||||
provisioner: nvmeof.csi.ceph.com
|
||||
provisioner: rook-ceph.nvmeof.csi.ceph.com
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: Immediate
|
||||
allowVolumeExpansion: true
|
||||
@@ -2,7 +2,7 @@
|
||||
gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v20240513-v0.1.0-35-gefb3255
|
||||
quay.io/ceph/ceph:v19.2.3
|
||||
quay.io/ceph/cosi:v0.1.2
|
||||
quay.io/cephcsi/ceph-csi-operator:v0.5.0
|
||||
quay.io/cephcsi/ceph-csi-operator:v0.6.0
|
||||
quay.io/cephcsi/cephcsi:v3.16.2
|
||||
quay.io/csiaddons/k8s-sidecar:v0.14.0
|
||||
registry.k8s.io/sig-storage/csi-attacher:v4.11.0
|
||||
|
||||
@@ -104,6 +104,8 @@ users:
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-nfs-ctrlplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-nfs-nodeplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-nvmeof-ctrlplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-nvmeof-nodeplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-rbd-ctrlplugin-sa # serviceaccount:namespace:operator
|
||||
- system:serviceaccount:rook-ceph:ceph-csi-rbd-nodeplugin-sa # serviceaccount:namespace:operator
|
||||
---
|
||||
|
||||
@@ -755,10 +755,6 @@ function test_csi_nfs_workload {
|
||||
function test_csi_nvmeof_workload {
|
||||
cd "${REPO_DIR}/deploy/examples/csi/nvmeof"
|
||||
|
||||
local cephcsi_image="${1:-quay.io/nixpanic/cephcsi:nvmeof}"
|
||||
local gateway_svc="rook-ceph-nvmeof-nvmeof-a"
|
||||
local gateway_listener_hostname="rook-ceph-nvmeof-nvmeof-a"
|
||||
local service_ip
|
||||
local old_gateway_pod
|
||||
|
||||
sed -i 's/failureDomain: .*/failureDomain: osd/' nvmeof-pool.yaml
|
||||
@@ -769,52 +765,47 @@ function test_csi_nvmeof_workload {
|
||||
kubectl create -f "${REPO_DIR}/deploy/examples/nvmeof-test.yaml"
|
||||
wait_for cephnvmeofgateway nvmeof rook-ceph 600
|
||||
timeout 300 bash <<EOF
|
||||
until kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {print \$3}' | grep -q "^Running$"; do
|
||||
until kubectl -n rook-ceph get pod --no-headers 2>/dev/null | awk '/rook-ceph-nvmeof-nvmeof-a-/ && \$3 == "Running" {f=1} END {exit !f}'; do
|
||||
echo "waiting for NVMe-oF gateway pod to be running"
|
||||
sleep 5
|
||||
done
|
||||
EOF
|
||||
|
||||
service_ip="$(kubectl -n rook-ceph get svc "${gateway_svc}" -o jsonpath='{.spec.clusterIP}')"
|
||||
if [[ -z "${service_ip}" ]]; then
|
||||
echo "failed to discover gateway service IP for ${gateway_svc}"
|
||||
exit 1
|
||||
# ceph-csi-operator creates these deployments, make sure they are available
|
||||
|
||||
timeout 300 bash <<EOF
|
||||
until kubectl -n rook-ceph get deployment --no-headers 2>/dev/null | grep -q "nvmeof"; do
|
||||
echo "waiting for CSI operator to create NVMe-oF controller plugin deployment"
|
||||
sleep 5
|
||||
done
|
||||
EOF
|
||||
|
||||
kubectl -n rook-ceph wait --for=condition=available \
|
||||
deployment/rook-ceph.nvmeof.csi.ceph.com-ctrlplugin --timeout=300s
|
||||
|
||||
timeout 300 bash <<EOF
|
||||
until kubectl -n rook-ceph get daemonset --no-headers 2>/dev/null | grep -q "nvmeof"; do
|
||||
echo "waiting for CSI operator to create NVMe-oF node plugin daemonset"
|
||||
sleep 5
|
||||
done
|
||||
EOF
|
||||
|
||||
kubectl -n rook-ceph rollout status \
|
||||
daemonset/rook-ceph.nvmeof.csi.ceph.com-nodeplugin --timeout=300s
|
||||
|
||||
lsmod | grep -E 'nvme(_|-)(tcp|fabrics)' || true
|
||||
if [[ ! -d /sys/module/nvme_tcp ]]; then
|
||||
echo "nvme_tcp kernel module is required but unavailable on kernel $(uname -r)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
sed -i "s|nvmeofGatewayAddress: \".*\"|nvmeofGatewayAddress: \"${service_ip}\"|g" storageclass.yaml
|
||||
# ceph-csi PR 6061 expects listeners to include hostname only.
|
||||
sed -i '/"address":/d' storageclass.yaml
|
||||
sed -i '/"port":/d' storageclass.yaml
|
||||
sed -i "s|\"hostname\": \".*\"|\"hostname\": \"${gateway_listener_hostname}\"|g" storageclass.yaml
|
||||
sed -i "s|image: quay.io/cephcsi/cephcsi:.*|image: ${cephcsi_image}|g" provisioner.yaml
|
||||
sed -i "s|image: quay.io/cephcsi/cephcsi:.*|image: ${cephcsi_image}|g" node-plugin.yaml
|
||||
|
||||
kubectl create -f provisioner.yaml
|
||||
kubectl -n rook-ceph wait --for=condition=available deployment/csi-nvmeofplugin-provisioner --timeout=300s
|
||||
|
||||
kubectl create -f storageclass.yaml
|
||||
kubectl create -f pvc.yaml
|
||||
if ! kubectl wait pvc/nvmeof-external-volume --for=jsonpath='{.status.phase}'=Bound --timeout=300s; then
|
||||
echo "PVC nvmeof-external-volume did not reach Bound; collecting diagnostics..."
|
||||
kubectl describe pvc nvmeof-external-volume || true
|
||||
kubectl get events -n default --sort-by=.metadata.creationTimestamp | tail -n 50 || true
|
||||
kubectl -n rook-ceph logs deploy/csi-nvmeofplugin-provisioner -c csi-provisioner --tail=200 || true
|
||||
kubectl -n rook-ceph logs deploy/csi-nvmeofplugin-provisioner -c csi-nvmeofplugin --tail=200 || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Sanity check module availability; prerequisites are installed earlier in CI setup.
|
||||
lsmod | grep -E 'nvme(_|-)(tcp|fabrics)' || true
|
||||
if [[ ! -d /sys/module/nvme_tcp ]]; then
|
||||
echo "nvme_tcp kernel module is required but unavailable on kernel $(uname -r)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
kubectl create -f node-plugin.yaml
|
||||
if ! kubectl -n rook-ceph rollout status daemonset/nvmeof.csi.ceph.com-nodeplugin --timeout=300s; then
|
||||
kubectl -n rook-ceph get pods -l app=nvmeof.csi.ceph.com-nodeplugin -o wide || true
|
||||
kubectl -n rook-ceph describe daemonset nvmeof.csi.ceph.com-nodeplugin || true
|
||||
echo "nvmeof nodeplugin daemonset failed to become ready"
|
||||
kubectl -n rook-ceph logs -l "app.kubernetes.io/part-of=rook-ceph.nvmeof.csi.ceph.com" --tail=200 || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -824,10 +815,10 @@ EOF
|
||||
kubectl -n default exec pod/nvmeof-test-pod -- sh -c "echo 'abcd' > /mnt/nvmeof/test.txt && cat /mnt/nvmeof/test.txt"
|
||||
kubectl -n default exec pod/nvmeof-test-pod -- sh -c "grep -qx 'abcd' /mnt/nvmeof/test.txt"
|
||||
|
||||
old_gateway_pod="$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {print $1; exit}')"
|
||||
old_gateway_pod="$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ {if (!f) f=$1} END {print f}')"
|
||||
kubectl -n rook-ceph delete pod "${old_gateway_pod}"
|
||||
timeout 300 bash <<EOF
|
||||
until new_gateway_pod=\$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ && \$3 == "Running" {print \$1; exit}') && [[ -n "\${new_gateway_pod}" && "\${new_gateway_pod}" != "${old_gateway_pod}" ]]; do
|
||||
until new_gateway_pod=\$(kubectl -n rook-ceph get pod --no-headers | awk '/rook-ceph-nvmeof-nvmeof-a-/ && \$3 == "Running" {if (!f) f=\$1} END {print f}') && [[ -n "\${new_gateway_pod}" && "\${new_gateway_pod}" != "${old_gateway_pod}" ]]; do
|
||||
echo "waiting for NVMe-oF gateway pod restart"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
Reference in new issue
Block a user