From 4dc19faa4b2e414771473e698bd1386d2a7fb19c Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 4 Mar 2026 17:07:09 +0100 Subject: [PATCH] nvmeof: update the ceph-csi-operator example to deploy Ceph-CSI/NVMe-oF There is no need to manually deploy Ceph-CSI/NVMe-oF anymore, the YAMLs can be removed as the example operator.yaml includes the options now. Signed-off-by: Niels de Vos (cherry picked from commit 02ed13afa0fc75dbc960a69f8713759c1300bc0e) --- deploy/examples/csi/nvmeof/driver.yaml | 8 + deploy/examples/csi/nvmeof/node-plugin.yaml | 202 ----------- deploy/examples/csi/nvmeof/provisioner.yaml | 339 ------------------- deploy/examples/csi/nvmeof/storageclass.yaml | 4 +- 4 files changed, 10 insertions(+), 543 deletions(-) create mode 100644 deploy/examples/csi/nvmeof/driver.yaml delete mode 100644 deploy/examples/csi/nvmeof/node-plugin.yaml delete mode 100644 deploy/examples/csi/nvmeof/provisioner.yaml diff --git a/deploy/examples/csi/nvmeof/driver.yaml b/deploy/examples/csi/nvmeof/driver.yaml new file mode 100644 index 000000000..4caa02ebe --- /dev/null +++ b/deploy/examples/csi/nvmeof/driver.yaml @@ -0,0 +1,8 @@ +apiVersion: csi.ceph.io/v1 +kind: Driver +metadata: + name: rook-ceph.nvmeof.csi.ceph.com + namespace: rook-ceph # namespace:operator +spec: + controllerPlugin: + hostNetwork: false diff --git a/deploy/examples/csi/nvmeof/node-plugin.yaml b/deploy/examples/csi/nvmeof/node-plugin.yaml deleted file mode 100644 index 2ad2fe41a..000000000 --- a/deploy/examples/csi/nvmeof/node-plugin.yaml +++ /dev/null @@ -1,202 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nvmeof.csi.ceph.com-nodeplugin - namespace: rook-ceph -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: nvmeof.csi.ceph.com-nodeplugin - template: - metadata: - labels: - app: nvmeof.csi.ceph.com-nodeplugin - spec: - containers: - - args: - - --v=5 - - --type=nvmeof - - --nodeserver=true - - --nodeid=$(NODE_ID) - - --drivername=nvmeof.csi.ceph.com - - --endpoint=unix:///csi/csi.sock - - --pidlimit=-1 - - --csi-addons-endpoint=unix:///csi/csi-addons.sock - - --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/ - - --logtostderr=false - - --alsologtostderr=true - - --log_file=csi-logs/csi-nvmeofplugin.log - env: - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: quay.io/cephcsi/cephcsi:v3.16.2 - imagePullPolicy: Always - name: csi-nvmeofplugin - resources: {} - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - drop: - - All - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /dev - name: host-dev - - mountPath: /sys - name: host-sys - - mountPath: /run/mount - name: host-run-mount - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /tmp/csi/keys - name: keys-tmp-dir - - mountPath: /csi - name: plugin-dir - # FIXED: Mount path should end with / to mount as directory - - mountPath: /etc/ceph-csi-config/ - name: ceph-csi-config - - mountPath: /var/lib/kubelet/plugins - mountPropagation: Bidirectional - name: plugin-mount-dir - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: pods-mount-dir - - mountPath: /etc/selinux - name: etc-selinux - readOnly: true - - mountPath: /run/secrets/tokens - name: oidc-token - readOnly: true - - mountPath: /csi-logs - name: logs-dir - # Log rotation mount - commented out since ConfigMap is optional - # - mountPath: /etc/logrotate.d - # name: log-rotate-dir - # readOnly: true - - args: - - "--v=1" - - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock" - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 - imagePullPolicy: IfNotPresent - name: driver-registrar - resources: {} - securityContext: - capabilities: - drop: - - All - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - hostPID: true - priorityClassName: system-node-critical - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - # FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend - # This service account already exists and has the necessary permissions - serviceAccount: rook-csi-rbd-plugin-sa - serviceAccountName: rook-csi-rbd-plugin-sa - terminationGracePeriodSeconds: 30 - # tolerations: - # - effect: NoSchedule - # key: node.ocs.openshift.io/storage - # operator: Equal - # value: "true" - volumes: - - hostPath: - path: /dev - type: "" - name: host-dev - - hostPath: - path: /sys - type: "" - name: host-sys - - hostPath: - path: /run/mount - type: "" - name: host-run-mount - - hostPath: - path: /lib/modules - type: "" - name: lib-modules - - emptyDir: - medium: Memory - name: keys-tmp-dir - # FIXED: Use rook-ceph-csi-config instead of ceph-csi-config - # This ConfigMap contains the cluster configuration with monitor endpoints - - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - name: ceph-csi-config - - hostPath: - path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins - type: Directory - name: plugin-mount-dir - - hostPath: - path: /var/lib/kubelet/pods - type: Directory - name: pods-mount-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - name: registration-dir - - hostPath: - path: /etc/selinux - type: "" - name: etc-selinux - - name: oidc-token - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: ceph-csi-kms - expirationSeconds: 3600 - path: oidc-token - - hostPath: - path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin - type: DirectoryOrCreate - name: logs-dir - # Log rotation ConfigMap - optional, can be created if log rotation is needed - # - configMap: - # defaultMode: 420 - # name: nvmeof.csi.ceph.com-logrotate-config - # name: log-rotate-dir - updateStrategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate diff --git a/deploy/examples/csi/nvmeof/provisioner.yaml b/deploy/examples/csi/nvmeof/provisioner.yaml deleted file mode 100644 index a3f49241f..000000000 --- a/deploy/examples/csi/nvmeof/provisioner.yaml +++ /dev/null @@ -1,339 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: nvmeof-csi-provisioner - # replace with non-default namespace name - namespace: rook-ceph - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nvmeof-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "patch", "update"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update", "patch", "create"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["get", "list", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["get"] - - apiGroups: [""] - resources: ["serviceaccounts/token"] - verbs: ["create"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["groupsnapshot.storage.k8s.io"] - resources: ["volumegroupsnapshotcontents/status"] - verbs: ["update", "patch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationcontents"] - verbs: ["get", "list", "watch"] - - apiGroups: ["replication.storage.openshift.io"] - resources: ["volumegroupreplicationclasses"] - verbs: ["get"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattributesclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nvmeof-csi-provisioner-role -subjects: - - kind: ServiceAccount - name: nvmeof-csi-provisioner - # replace with non-default namespace name - namespace: rook-ceph -roleRef: - kind: ClusterRole - name: nvmeof-external-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - # replace with non-default namespace name - namespace: rook-ceph - name: nvmeof-external-provisioner-cfg -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nvmeof-csi-provisioner-role-cfg - # replace with non-default namespace name - namespace: rook-ceph -subjects: - - kind: ServiceAccount - name: nvmeof-csi-provisioner - # replace with non-default namespace name - namespace: rook-ceph -roleRef: - kind: Role - name: nvmeof-external-provisioner-cfg - apiGroup: rbac.authorization.k8s.io -# --- -# # ssc based on the Ceph-CSI ssc -# # the "users:" contains a reference to the "rook-ceph" namespace -# # -# kind: SecurityContextConstraints -# apiVersion: security.openshift.io/v1 -# metadata: -# name: "ceph-nvmeof-provisioner" -# # To allow running privilegedContainers -# allowPrivilegedContainer: true -# # SYS_ADMIN is needed for rbd to execute rbd map command -# allowedCapabilities: -# - "SYS_ADMIN" -# # Set to false as we write to RootFilesystem inside container -# readOnlyRootFilesystem: false -# requiredDropCapabilities: -# - ALL -# # The type of volumes which are mounted to csi pods -# volumes: -# - configMap -# - projected -# - emptyDir -# runAsUser: -# type: RunAsAny -# seLinuxContext: -# type: RunAsAny -# users: -# # A user needs to be added for each service account. -# - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner" ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: csi-nvmeofplugin-provisioner - # replace with non-default namespace name - namespace: rook-ceph -spec: - replicas: 1 - selector: - matchLabels: - app: csi-nvmeofplugin-provisioner - template: - metadata: - annotations: - openshift.io/scc: rook-ceph-csi - labels: - app: csi-nvmeofplugin-provisioner - spec: - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - csi-nvmeofplugin-provisioner - topologyKey: "kubernetes.io/hostname" - serviceAccountName: nvmeof-csi-provisioner - priorityClassName: system-cluster-critical - containers: - - name: csi-nvmeofplugin - # for stable functionality replace canary with latest release version - # image: quay.io/gdidi/cephcsi:nvmeof - image: quay.io/cephcsi/cephcsi:v3.16.2 - command: ["/usr/local/bin/cephcsi"] - args: - - "--nodeid=$(NODE_ID)" - - "--type=nvmeof" - - "--controllerserver=true" - - "--endpoint=$(CSI_ENDPOINT)" - - "--v=5" - - "--drivername=nvmeof.csi.ceph.com" - - "--pidlimit=-1" - - "--rbdhardmaxclonedepth=8" - - "--rbdsoftmaxclonedepth=4" - - "--enableprofiling=false" - - "--setmetadata=true" - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CSI_ENDPOINT - value: unix:///csi/csi-provisioner.sock - imagePullPolicy: "IfNotPresent" - securityContext: - privileged: true - capabilities: - drop: - - ALL - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: ceph-csi-config - mountPath: /etc/ceph-csi-config/ - - name: keys-tmp-dir - mountPath: /tmp/csi/keys - - name: oidc-token - mountPath: /run/secrets/tokens - readOnly: true - - name: csi-provisioner - # image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0 - image: quay.io/nixpanic/csi-provisioner:pr1440 - args: - - "--csi-address=$(ADDRESS)" - - "--v=1" - - "--timeout=150s" - - "--retry-interval-start=500ms" - - "--leader-election=true" - - "--feature-gates=HonorPVReclaimPolicy=true" - - "--prevent-volume-mode-conversion=true" - # if fstype is not specified in storageclass, ext4 is default - - "--default-fstype=ext4" - - "--extra-create-metadata=true" - - "--immediate-topology=false" - - "--http-endpoint=$(POD_IP):8090" - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 8090 - name: provisioner - protocol: TCP - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-attacher - image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0 - args: - - "--v=1" - - "--csi-address=$(ADDRESS)" - - "--leader-election=true" - - "--retry-interval-start=500ms" - - "--default-fstype=ext4" - - "--http-endpoint=$(POD_IP):8093" - env: - - name: ADDRESS - value: /csi/csi-provisioner.sock - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - imagePullPolicy: "IfNotPresent" - ports: - - containerPort: 8093 - name: attacher - protocol: TCP - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: csi-resizer - image: quay.io/nixpanic/csi-resizer:pr544 - # image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0 - args: - - "--csi-address=$(ADDRESS)" - - "--v=5" - - "--leader-election=true" - - "--handle-volume-inuse-error=false" - - "--feature-gates=VolumeAttributesClass=true" - env: - - name: ADDRESS - value: unix:///csi/csi-provisioner.sock - volumeMounts: - - name: socket-dir - mountPath: /csi - volumes: - - name: socket-dir - emptyDir: - medium: "Memory" - # FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config - # This ConfigMap contains the cluster configuration with monitor endpoints - # that Ceph-CSI needs to connect to the Ceph cluster - - name: ceph-csi-config - configMap: - name: rook-ceph-csi-config - items: - - key: csi-cluster-config-json - path: config.json - - name: keys-tmp-dir - emptyDir: - medium: "Memory" - # Only if nvmeof support encryption - - name: oidc-token - projected: - sources: - - serviceAccountToken: - path: oidc-token - expirationSeconds: 3600 - audience: ceph-csi-kms diff --git a/deploy/examples/csi/nvmeof/storageclass.yaml b/deploy/examples/csi/nvmeof/storageclass.yaml index 2f6691ffd..ffe19a569 100644 --- a/deploy/examples/csi/nvmeof/storageclass.yaml +++ b/deploy/examples/csi/nvmeof/storageclass.yaml @@ -7,7 +7,7 @@ parameters: pool: nvmeof subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph # Management API - talks to gateway to create subsystems/namespaces - nvmeofGatewayAddress: "172.30.47.120" + nvmeofGatewayAddress: "rook-ceph-nvmeof-nvmeof-a.rook-ceph.svc.cluster.local" nvmeofGatewayPort: "5500" # Data Plane - worker nodes connect here for actual I/O # List ALL gateway pods for HA and multipath @@ -27,7 +27,7 @@ parameters: csi.storage.k8s.io/node-expand-secret-namespace: rook-ceph imageFormat: "2" imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff -provisioner: nvmeof.csi.ceph.com +provisioner: rook-ceph.nvmeof.csi.ceph.com reclaimPolicy: Delete volumeBindingMode: Immediate allowVolumeExpansion: true