################################################################################################################# # Create the rook operator and necessary security context constraints for running # Rook in an OpenShift cluster. # For example, to create the rook-ceph cluster: # oc create -f crds.yaml -f common.yaml -f operator-openshift.yaml # oc create -f cluster.yaml ################################################################################################################# # scc for the Rook and Ceph daemons kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: name: rook-ceph allowPrivilegedContainer: true allowHostDirVolumePlugin: true allowHostPID: false # set to true if running rook with host networking enabled allowHostNetwork: false # set to true if running rook with the provider as host allowHostPorts: false priority: allowedCapabilities: ["MKNOD", "SYS_ADMIN"] allowHostIPC: true readOnlyRootFilesystem: false # drop all default privileges requiredDropCapabilities: ["All"] defaultAddCapabilities: [] runAsUser: type: RunAsAny seLinuxContext: type: MustRunAs fsGroup: type: MustRunAs supplementalGroups: type: RunAsAny volumes: - configMap - downwardAPI - emptyDir - hostPath - persistentVolumeClaim - projected - secret users: # A user needs to be added for each rook service account. # This assumes running in the default sample "rook-ceph" namespace. # If other namespaces or service accounts are configured, they need to be updated here. - system:serviceaccount:rook-ceph:rook-ceph-system # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:rook-ceph-default # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-rgw # serviceaccount:namespace:cluster - system:serviceaccount:rook-ceph:rook-ceph-nvmeof # serviceaccount:namespace:cluster --- # scc for the CSI driver kind: SecurityContextConstraints apiVersion: security.openshift.io/v1 metadata: name: rook-ceph-csi # To allow running privilegedContainers allowPrivilegedContainer: true # CSI daemonset pod needs hostnetworking allowHostNetwork: true # This need to be set to true as we use HostPath allowHostDirVolumePlugin: true priority: # SYS_ADMIN is needed for rbd to execute rbd map command allowedCapabilities: ["SYS_ADMIN"] # Needed as we run liveness container on daemonset pods allowHostPorts: true # Needed as we are setting this in RBD plugin pod allowHostPID: true # Required for encryption allowHostIPC: true # Set to false as we write to RootFilesystem inside csi containers readOnlyRootFilesystem: false runAsUser: type: RunAsAny seLinuxContext: type: RunAsAny fsGroup: type: RunAsAny supplementalGroups: type: RunAsAny # The type of volumes which are mounted to csi pods volumes: - configMap - emptyDir - hostPath - projected users: # A user needs to be added for each csi service account managed by the ceph-csi-operator. # This assumes running in the default sample "rook-ceph" namespace. # If other namespaces or service accounts are configured, they need to be updated here. - system:serviceaccount:rook-ceph:ceph-csi-cephfs-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-cephfs-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-nfs-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-nfs-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-nvmeof-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-nvmeof-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-rbd-ctrlplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:ceph-csi-rbd-nodeplugin-sa # serviceaccount:namespace:operator - system:serviceaccount:rook-ceph:nvmeof-csi-provisioner # serviceaccount:namespace:operator --- # Rook Ceph Operator Config # Use this ConfigMap to override operator configurations # Precedence will be given to this config in case Env Var also exists for the same # kind: ConfigMap apiVersion: v1 metadata: name: rook-ceph-operator-config # should be in the namespace of the operator namespace: rook-ceph # namespace:operator data: # The logging level for the operator: ERROR | WARNING | INFO | DEBUG ROOK_LOG_LEVEL: "INFO" # Allow using loop devices for osds in test clusters. ROOK_CEPH_ALLOW_LOOP_DEVICES: "false" # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" # Custom prefix value for the OBC provisioner instead of ceph cluster namespace, do not set on existing cluster # ROOK_OBC_PROVISIONER_NAME_PREFIX: "custom-prefix" # Many OBC additional config fields may be risky for administrators to allow users control over. # The safe and default-allowed fields are 'maxObjects' and 'maxSize'. # Other fields should be considered risky. To allow all additional configs, use this value: # "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner" # ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. ROOK_ENABLE_DISCOVERY_DAEMON: "false" # The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15. ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" # Rook Discover toleration. Will tolerate all taints with all keys. # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. # DISCOVER_TOLERATIONS: | # - effect: NoSchedule # key: node-role.kubernetes.io/control-plane # operator: Exists # - effect: NoExecute # key: node-role.kubernetes.io/etcd # operator: Exists # (Optional) Rook Discover priority class name to set on the pod(s) # DISCOVER_PRIORITY_CLASS_NAME: "" # (Optional) Discover Agent NodeAffinity. # DISCOVER_AGENT_NODE_AFFINITY: | # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: # - matchExpressions: # - key: myKey # operator: DoesNotExist # (Optional) Discover Agent Pod Labels. # DISCOVER_AGENT_POD_LABELS: "key1=value1,key2=value2" # Disable automatic orchestration when new devices are discovered ROOK_DISABLE_DEVICE_HOTPLUG: "false" # The duration between discovering devices in the rook-discover daemonset. ROOK_DISCOVER_DEVICES_INTERVAL: "60m" # DISCOVER_DAEMON_RESOURCES: | # - name: DISCOVER_DAEMON_RESOURCES # resources: # limits: # memory: 512Mi # requests: # cpu: 200m # memory: 128Mi # Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled ROOK_ENFORCE_HOST_NETWORK: "false" # RevisionHistoryLimit value for all deployments created by rook. # ROOK_REVISION_HISTORY_LIMIT: "3" --- # CSI operator resources managed by the admin. # These CRs define the configuration for CSI drivers deployed by the ceph-csi-operator. # Customize these as needed for your environment. # For the complete list of ceph-csi-operator chart settings, see: # https://github.com/ceph/ceph-csi-operator/blob/main/docs/helm-charts/drivers-chart.md#configuration # ImageSet ConfigMap defines the container images used by the CSI drivers. apiVersion: v1 kind: ConfigMap metadata: name: rook-csi-operator-image-set-configmap namespace: rook-ceph # namespace:operator data: provisioner: "registry.k8s.io/sig-storage/csi-provisioner:v6.2.0" attacher: "registry.k8s.io/sig-storage/csi-attacher:v4.12.0" resizer: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0" snapshotter: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0" registrar: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.17.0" plugin: "quay.io/cephcsi/cephcsi:v3.17.0" addons: "quay.io/csiaddons/k8s-sidecar:v0.14.0" --- # OperatorConfig defines the default settings for all CSI drivers. apiVersion: csi.ceph.io/v1 kind: OperatorConfig metadata: name: ceph-csi-operator-config namespace: rook-ceph # namespace:operator spec: driverSpecDefaults: log: verbosity: 0 imageSet: name: rook-csi-operator-image-set-configmap enableMetadata: true generateOMapInfo: false fsGroupPolicy: File deployCsiAddons: false cephFsClientType: kernel nodePlugin: priorityClassName: "system-node-critical" kubeletDirPath: /var/lib/kubelet enableSeLinuxHostMount: false controllerPlugin: priorityClassName: "system-cluster-critical" deploymentStrategy: type: Recreate --- # RBD CSI Driver apiVersion: csi.ceph.io/v1 kind: Driver metadata: name: rook-ceph.rbd.csi.ceph.com # csi-provisioner-name namespace: rook-ceph # namespace:operator spec: fsGroupPolicy: File nodePlugin: updateStrategy: type: RollingUpdate controllerPlugin: {} --- # CephFS CSI Driver apiVersion: csi.ceph.io/v1 kind: Driver metadata: name: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name namespace: rook-ceph # namespace:operator spec: fsGroupPolicy: File cephFsClientType: kernel nodePlugin: updateStrategy: type: RollingUpdate controllerPlugin: {} --- # The deployment for the rook operator # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 kind: Deployment metadata: name: rook-ceph-operator namespace: rook-ceph # namespace:operator labels: operator: rook storage-backend: ceph app.kubernetes.io/name: rook-ceph app.kubernetes.io/instance: rook-ceph app.kubernetes.io/component: rook-ceph-operator app.kubernetes.io/part-of: rook-ceph-operator spec: selector: matchLabels: app: rook-ceph-operator strategy: type: Recreate replicas: 1 template: metadata: labels: app: rook-ceph-operator spec: tolerations: - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 5 serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator image: docker.io/rook/ceph:master args: ["ceph", "operator"] securityContext: runAsNonRoot: true runAsUser: 2016 runAsGroup: 2016 volumeMounts: - mountPath: /var/lib/rook name: rook-config - mountPath: /etc/ceph name: default-config-dir env: - name: ROOK_CURRENT_NAMESPACE_ONLY value: "false" # Whether to start pods as privileged that mount a host path, which includes the Ceph mon, osd pods and csi provisioners(if logrotation is on). # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED value: "true" # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". # In case of more than one regex, use comma to separate between them. # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" # add regex expression after putting a comma to blacklist a disk # If value is empty, the default regex will be used. - name: DISCOVER_DAEMON_UDEV_BLACKLIST value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" # Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs. - name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET value: "false" # - name: DISCOVER_DAEMON_RESOURCES # value: | # resources: # limits: # memory: 512Mi # requests: # cpu: 100m # memory: 128Mi # Time to wait until the node controller will move Rook pods to other # nodes after detecting an unreachable node. # Pods affected by this setting are: # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox # The value used in this variable replaces the default value of 300 secs # added automatically by k8s as Toleration for # # The total amount of time to reschedule Rook pods in healthy nodes # before detecting a condition will be the sum of: # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS value: "5" # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName # The pod name to pass with the downward API - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name # The pod namespace to pass with the downward API - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace # Recommended resource requests and limits, if desired #resources: # limits: # memory: 512Mi # requests: # cpu: 100m # memory: 128Mi # Uncomment it to run lib bucket provisioner in multithreaded mode #- name: LIB_BUCKET_PROVISIONER_THREADS # value: "5" # Uncomment these two settings to run the operator on the host network # hostNetwork: true # dnsPolicy: ClusterFirstWithHostNet volumes: - name: rook-config emptyDir: {} - name: default-config-dir emptyDir: {} # OLM: END OPERATOR DEPLOYMENT