nvmeof: update the ceph-csi-operator example to deploy Ceph-CSI/NVMe-oF

There is no need to manually deploy Ceph-CSI/NVMe-oF anymore, the YAMLs can be removed as the example operator.yaml includes the options now. Signed-off-by: Niels de Vos <ndevos@ibm.com> (cherry picked from commit 02ed13afa0)
2026-03-24 19:44:27 +00:00
1 parent 945bda73fc
commit 4dc19faa4b
@@ -0,0 +1,8 @@
+apiVersion: csi.ceph.io/v1
+kind: Driver
+metadata:
+  name: rook-ceph.nvmeof.csi.ceph.com
+  namespace: rook-ceph # namespace:operator
+spec:
+  controllerPlugin:
+    hostNetwork: false
@@ -1,202 +0,0 @@
---
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: nvmeof.csi.ceph.com-nodeplugin
-  namespace: rook-ceph
-spec:
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      app: nvmeof.csi.ceph.com-nodeplugin
-  template:
-    metadata:
-      labels:
-        app: nvmeof.csi.ceph.com-nodeplugin
-    spec:
-      containers:
-        - args:
-            - --v=5
-            - --type=nvmeof
-            - --nodeserver=true
-            - --nodeid=$(NODE_ID)
-            - --drivername=nvmeof.csi.ceph.com
-            - --endpoint=unix:///csi/csi.sock
-            - --pidlimit=-1
-            - --csi-addons-endpoint=unix:///csi/csi-addons.sock
-            - --stagingpath=/var/lib/kubelet/plugins/kubernetes.io/csi/
-            - --logtostderr=false
-            - --alsologtostderr=true
-            - --log_file=csi-logs/csi-nvmeofplugin.log
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: metadata.namespace
-          image: quay.io/cephcsi/cephcsi:v3.16.2
-          imagePullPolicy: Always
-          name: csi-nvmeofplugin
-          resources: {}
-          securityContext:
-            allowPrivilegeEscalation: true
-            capabilities:
-              add:
-                - SYS_ADMIN
-              drop:
-                - All
-            privileged: true
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          volumeMounts:
-            - mountPath: /dev
-              name: host-dev
-            - mountPath: /sys
-              name: host-sys
-            - mountPath: /run/mount
-              name: host-run-mount
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: true
-            - mountPath: /tmp/csi/keys
-              name: keys-tmp-dir
-            - mountPath: /csi
-              name: plugin-dir
-            # FIXED: Mount path should end with / to mount as directory
-            - mountPath: /etc/ceph-csi-config/
-              name: ceph-csi-config
-            - mountPath: /var/lib/kubelet/plugins
-              mountPropagation: Bidirectional
-              name: plugin-mount-dir
-            - mountPath: /var/lib/kubelet/pods
-              mountPropagation: Bidirectional
-              name: pods-mount-dir
-            - mountPath: /etc/selinux
-              name: etc-selinux
-              readOnly: true
-            - mountPath: /run/secrets/tokens
-              name: oidc-token
-              readOnly: true
-            - mountPath: /csi-logs
-              name: logs-dir
-          # Log rotation mount - commented out since ConfigMap is optional
-          # - mountPath: /etc/logrotate.d
-          #   name: log-rotate-dir
-          #   readOnly: true
-        - args:
-            - "--v=1"
-            - "--csi-address=/csi/csi.sock"
-            - "--kubelet-registration-path=/var/lib/kubelet/plugins/nvmeof.csi.ceph.com/csi.sock"
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0
-          imagePullPolicy: IfNotPresent
-          name: driver-registrar
-          resources: {}
-          securityContext:
-            capabilities:
-              drop:
-                - All
-            privileged: true
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          volumeMounts:
-            - mountPath: /csi
-              name: plugin-dir
-            - mountPath: /registration
-              name: registration-dir
-      dnsPolicy: ClusterFirstWithHostNet
-      hostNetwork: true
-      hostPID: true
-      priorityClassName: system-node-critical
-      restartPolicy: Always
-      schedulerName: default-scheduler
-      securityContext: {}
-      # FIXED: Use rook-csi-rbd-plugin-sa since nvmeof uses RBD backend
-      # This service account already exists and has the necessary permissions
-      serviceAccount: rook-csi-rbd-plugin-sa
-      serviceAccountName: rook-csi-rbd-plugin-sa
-      terminationGracePeriodSeconds: 30
-      # tolerations:
-      # - effect: NoSchedule
-      # key: node.ocs.openshift.io/storage
-      # operator: Equal
-      # value: "true"
-      volumes:
-        - hostPath:
-            path: /dev
-            type: ""
-          name: host-dev
-        - hostPath:
-            path: /sys
-            type: ""
-          name: host-sys
-        - hostPath:
-            path: /run/mount
-            type: ""
-          name: host-run-mount
-        - hostPath:
-            path: /lib/modules
-            type: ""
-          name: lib-modules
-        - emptyDir:
-            medium: Memory
-          name: keys-tmp-dir
-        # FIXED: Use rook-ceph-csi-config instead of ceph-csi-config
-        # This ConfigMap contains the cluster configuration with monitor endpoints
-        - configMap:
-            name: rook-ceph-csi-config
-            items:
-              - key: csi-cluster-config-json
-                path: config.json
-          name: ceph-csi-config
-        - hostPath:
-            path: /var/lib/kubelet/plugins/nvmeof.csi.ceph.com
-            type: DirectoryOrCreate
-          name: plugin-dir
-        - hostPath:
-            path: /var/lib/kubelet/plugins
-            type: Directory
-          name: plugin-mount-dir
-        - hostPath:
-            path: /var/lib/kubelet/pods
-            type: Directory
-          name: pods-mount-dir
-        - hostPath:
-            path: /var/lib/kubelet/plugins_registry
-            type: Directory
-          name: registration-dir
-        - hostPath:
-            path: /etc/selinux
-            type: ""
-          name: etc-selinux
-        - name: oidc-token
-          projected:
-            defaultMode: 420
-            sources:
-              - serviceAccountToken:
-                  audience: ceph-csi-kms
-                  expirationSeconds: 3600
-                  path: oidc-token
-        - hostPath:
-            path: /var/lib/cephcsi/nvmeof.csi.ceph.com-nodeplugin
-            type: DirectoryOrCreate
-          name: logs-dir
-      # Log rotation ConfigMap - optional, can be created if log rotation is needed
-      # - configMap:
-      #     defaultMode: 420
-      #     name: nvmeof.csi.ceph.com-logrotate-config
-      #   name: log-rotate-dir
-  updateStrategy:
-    rollingUpdate:
-      maxSurge: 0
-      maxUnavailable: 1
-    type: RollingUpdate
@@ -1,339 +0,0 @@
---
-kind: ServiceAccount
-apiVersion: v1
-metadata:
-  name: nvmeof-csi-provisioner
-  # replace with non-default namespace name
-  namespace: rook-ceph
-
---
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nvmeof-external-provisioner-runner
-rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["list", "watch", "create", "update", "patch"]
-  - apiGroups: ["events.k8s.io"]
-    resources: ["events"]
-    verbs: ["create", "patch", "update"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update", "patch", "create"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["get", "list", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments/status"]
-    verbs: ["patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts"]
-    verbs: ["get"]
-  - apiGroups: [""]
-    resources: ["serviceaccounts/token"]
-    verbs: ["create"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: ["groupsnapshot.storage.k8s.io"]
-    resources: ["volumegroupsnapshotcontents/status"]
-    verbs: ["update", "patch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationcontents"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["replication.storage.openshift.io"]
-    resources: ["volumegroupreplicationclasses"]
-    verbs: ["get"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattributesclasses"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims/status"]
-    verbs: ["patch"]
---
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nvmeof-csi-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: nvmeof-csi-provisioner
-    # replace with non-default namespace name
-    namespace: rook-ceph
-roleRef:
-  kind: ClusterRole
-  name: nvmeof-external-provisioner-runner
-  apiGroup: rbac.authorization.k8s.io
---
-kind: Role
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  # replace with non-default namespace name
-  namespace: rook-ceph
-  name: nvmeof-external-provisioner-cfg
-rules:
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "list", "watch", "create", "update", "delete"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["get", "watch", "list", "delete", "update", "create"]
---
-kind: RoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: nvmeof-csi-provisioner-role-cfg
-  # replace with non-default namespace name
-  namespace: rook-ceph
-subjects:
-  - kind: ServiceAccount
-    name: nvmeof-csi-provisioner
-    # replace with non-default namespace name
-    namespace: rook-ceph
-roleRef:
-  kind: Role
-  name: nvmeof-external-provisioner-cfg
-  apiGroup: rbac.authorization.k8s.io
-# ---
-# # ssc based on the Ceph-CSI ssc
-# # the "users:" contains a reference to the "rook-ceph" namespace
-# #
-# kind: SecurityContextConstraints
-# apiVersion: security.openshift.io/v1
-# metadata:
-#   name: "ceph-nvmeof-provisioner"
-# # To allow running privilegedContainers
-# allowPrivilegedContainer: true
-# # SYS_ADMIN is needed for rbd to execute rbd map command
-# allowedCapabilities:
-#   - "SYS_ADMIN"
-# # Set to false as we write to RootFilesystem inside container
-# readOnlyRootFilesystem: false
-# requiredDropCapabilities:
-#   - ALL
-# # The type of volumes which are mounted to csi pods
-# volumes:
-#   - configMap
-#   - projected
-#   - emptyDir
-# runAsUser:
-#   type: RunAsAny
-# seLinuxContext:
-#   type: RunAsAny
-# users:
-#   # A user needs to be added for each service account.
-#   - "system:serviceaccount:rook-ceph:nvmeof-csi-provisioner"
---
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  name: csi-nvmeofplugin-provisioner
-  # replace with non-default namespace name
-  namespace: rook-ceph
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: csi-nvmeofplugin-provisioner
-  template:
-    metadata:
-      annotations:
-        openshift.io/scc: rook-ceph-csi
-      labels:
-        app: csi-nvmeofplugin-provisioner
-    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - csi-nvmeofplugin-provisioner
-              topologyKey: "kubernetes.io/hostname"
-      serviceAccountName: nvmeof-csi-provisioner
-      priorityClassName: system-cluster-critical
-      containers:
-        - name: csi-nvmeofplugin
-          # for stable functionality replace canary with latest release version
-          # image: quay.io/gdidi/cephcsi:nvmeof
-          image: quay.io/cephcsi/cephcsi:v3.16.2
-          command: ["/usr/local/bin/cephcsi"]
-          args:
-            - "--nodeid=$(NODE_ID)"
-            - "--type=nvmeof"
-            - "--controllerserver=true"
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--v=5"
-            - "--drivername=nvmeof.csi.ceph.com"
-            - "--pidlimit=-1"
-            - "--rbdhardmaxclonedepth=8"
-            - "--rbdsoftmaxclonedepth=4"
-            - "--enableprofiling=false"
-            - "--setmetadata=true"
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: NODE_ID
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CSI_ENDPOINT
-              value: unix:///csi/csi-provisioner.sock
-          imagePullPolicy: "IfNotPresent"
-          securityContext:
-            privileged: true
-            capabilities:
-              drop:
-                - ALL
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-            - name: ceph-csi-config
-              mountPath: /etc/ceph-csi-config/
-            - name: keys-tmp-dir
-              mountPath: /tmp/csi/keys
-            - name: oidc-token
-              mountPath: /run/secrets/tokens
-              readOnly: true
-        - name: csi-provisioner
-          #          image: registry.k8s.io/sig-storage/csi-provisioner:v5.1.0
-          image: quay.io/nixpanic/csi-provisioner:pr1440
-          args:
-            - "--csi-address=$(ADDRESS)"
-            - "--v=1"
-            - "--timeout=150s"
-            - "--retry-interval-start=500ms"
-            - "--leader-election=true"
-            - "--feature-gates=HonorPVReclaimPolicy=true"
-            - "--prevent-volume-mode-conversion=true"
-            # if fstype is not specified in storageclass, ext4 is default
-            - "--default-fstype=ext4"
-            - "--extra-create-metadata=true"
-            - "--immediate-topology=false"
-            - "--http-endpoint=$(POD_IP):8090"
-          env:
-            - name: ADDRESS
-              value: unix:///csi/csi-provisioner.sock
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-          imagePullPolicy: "IfNotPresent"
-          ports:
-            - containerPort: 8090
-              name: provisioner
-              protocol: TCP
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-        - name: csi-attacher
-          image: registry.k8s.io/sig-storage/csi-attacher:v4.8.0
-          args:
-            - "--v=1"
-            - "--csi-address=$(ADDRESS)"
-            - "--leader-election=true"
-            - "--retry-interval-start=500ms"
-            - "--default-fstype=ext4"
-            - "--http-endpoint=$(POD_IP):8093"
-          env:
-            - name: ADDRESS
-              value: /csi/csi-provisioner.sock
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-          imagePullPolicy: "IfNotPresent"
-          ports:
-            - containerPort: 8093
-              name: attacher
-              protocol: TCP
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-        - name: csi-resizer
-          image: quay.io/nixpanic/csi-resizer:pr544
-          #          image: registry.k8s.io/sig-storage/csi-resizer:v1.12.0
-          args:
-            - "--csi-address=$(ADDRESS)"
-            - "--v=5"
-            - "--leader-election=true"
-            - "--handle-volume-inuse-error=false"
-            - "--feature-gates=VolumeAttributesClass=true"
-          env:
-            - name: ADDRESS
-              value: unix:///csi/csi-provisioner.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-      volumes:
-        - name: socket-dir
-          emptyDir:
-            medium: "Memory"
-        # FIXED: Mount rook-ceph-csi-config instead of rook-ceph-nvmeof-*-config
-        # This ConfigMap contains the cluster configuration with monitor endpoints
-        # that Ceph-CSI needs to connect to the Ceph cluster
-        - name: ceph-csi-config
-          configMap:
-            name: rook-ceph-csi-config
-            items:
-              - key: csi-cluster-config-json
-                path: config.json
-        - name: keys-tmp-dir
-          emptyDir:
-            medium: "Memory"
-        # Only if nvmeof support encryption
-        - name: oidc-token
-          projected:
-            sources:
-              - serviceAccountToken:
-                  path: oidc-token
-                  expirationSeconds: 3600
-                  audience: ceph-csi-kms
@@ -7,7 +7,7 @@ parameters:
  pool: nvmeof
  subsystemNQN: nqn.2016-06.io.spdk:cnode1.rook-ceph
  # Management API - talks to gateway to create subsystems/namespaces
-  nvmeofGatewayAddress: "172.30.47.120"
+  nvmeofGatewayAddress: "rook-ceph-nvmeof-nvmeof-a.rook-ceph.svc.cluster.local"
  nvmeofGatewayPort: "5500"
  # Data Plane - worker nodes connect here for actual I/O
  # List ALL gateway pods for HA and multipath
@@ -27,7 +27,7 @@ parameters:
  csi.storage.k8s.io/node-expand-secret-namespace: rook-ceph
  imageFormat: "2"
  imageFeatures: layering,deep-flatten,exclusive-lock,object-map,fast-diff
-provisioner: nvmeof.csi.ceph.com
+provisioner: rook-ceph.nvmeof.csi.ceph.com
 reclaimPolicy: Delete
 volumeBindingMode: Immediate
 allowVolumeExpansion: true