forked from rook/rook
Travis Nielsen
45ec32e513
The ROOK_RECONCILE_CONCURRENT_CLUSTERS feature was implemented in v1.19. This feature has been stable, with no related issues reported. The feature is tested in the CI with no known stability issues. Let's declare this feature as stable. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
297 lines
12 KiB
YAML
297 lines
12 KiB
YAML
#################################################################################################################
|
|
# The deployment for the rook operator
|
|
# Contains the common settings for most Kubernetes deployments.
|
|
# For example, to create the rook-ceph cluster:
|
|
# kubectl create -f crds.yaml -f common.yaml -f operator.yaml
|
|
# kubectl create -f cluster.yaml
|
|
#
|
|
# Also see other operator sample files for variations of operator.yaml:
|
|
# - operator-openshift.yaml: Common settings for running in OpenShift
|
|
###############################################################################################################
|
|
|
|
# Rook Ceph Operator Config ConfigMap
|
|
# Use this ConfigMap to override Rook-Ceph Operator configurations.
|
|
# NOTE! Precedence will be given to this config if the same Env Var config also exists in the
|
|
# Operator Deployment.
|
|
# To move a configuration(s) from the Operator Deployment to this ConfigMap, add the config
|
|
# here. It is recommended to then remove it from the Deployment to eliminate any future confusion.
|
|
kind: ConfigMap
|
|
apiVersion: v1
|
|
metadata:
|
|
name: rook-ceph-operator-config
|
|
# should be in the namespace of the operator
|
|
namespace: rook-ceph # namespace:operator
|
|
data:
|
|
# The logging level for the operator: ERROR | WARNING | INFO | DEBUG
|
|
ROOK_LOG_LEVEL: "INFO"
|
|
|
|
# The address for the operator's controller-runtime metrics. 0 is disabled. :8080 serves metrics on port 8080.
|
|
ROOK_OPERATOR_METRICS_BIND_ADDRESS: "0"
|
|
|
|
# Allow using loop devices for osds in test clusters.
|
|
ROOK_CEPH_ALLOW_LOOP_DEVICES: "false"
|
|
|
|
# Delete unused generated CRUSH rules after the mgr starts. Disable this if
|
|
# you need Rook to leave custom CRUSH rules in place.
|
|
ROOK_DELETE_UNUSED_CRUSH_RULES: "true"
|
|
|
|
# Whether the OBC provisioner should watch on the ceph cluster namespace or not, if not default provisioner value is set
|
|
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
|
|
|
|
# Custom prefix value for the OBC provisioner instead of ceph cluster namespace, do not set on existing cluster
|
|
# ROOK_OBC_PROVISIONER_NAME_PREFIX: "custom-prefix"
|
|
|
|
# Many OBC additional config fields may be risky for administrators to allow users control over.
|
|
# The safe and default-allowed fields are 'maxObjects' and 'maxSize'.
|
|
# Other fields should be considered risky. To allow all additional configs, use this value:
|
|
# "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"
|
|
# ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs
|
|
|
|
# Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
|
|
# This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
|
|
ROOK_ENABLE_DISCOVERY_DAEMON: "false"
|
|
# The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15.
|
|
ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15"
|
|
# Rook Discover toleration. Will tolerate all taints with all keys.
|
|
# (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format.
|
|
# DISCOVER_TOLERATIONS: |
|
|
# - effect: NoSchedule
|
|
# key: node-role.kubernetes.io/control-plane
|
|
# operator: Exists
|
|
# - effect: NoExecute
|
|
# key: node-role.kubernetes.io/etcd
|
|
# operator: Exists
|
|
# (Optional) Rook Discover priority class name to set on the pod(s)
|
|
# DISCOVER_PRIORITY_CLASS_NAME: "<PriorityClassName>"
|
|
# (Optional) Discover Agent NodeAffinity.
|
|
# DISCOVER_AGENT_NODE_AFFINITY: |
|
|
# requiredDuringSchedulingIgnoredDuringExecution:
|
|
# nodeSelectorTerms:
|
|
# - matchExpressions:
|
|
# - key: myKey
|
|
# operator: DoesNotExist
|
|
# (Optional) Discover Agent Pod Labels.
|
|
# DISCOVER_AGENT_POD_LABELS: "key1=value1,key2=value2"
|
|
# Disable automatic orchestration when new devices are discovered
|
|
ROOK_DISABLE_DEVICE_HOTPLUG: "false"
|
|
# The duration between discovering devices in the rook-discover daemonset.
|
|
ROOK_DISCOVER_DEVICES_INTERVAL: "60m"
|
|
# DISCOVER_DAEMON_RESOURCES: |
|
|
# - name: DISCOVER_DAEMON_RESOURCES
|
|
# resources:
|
|
# limits:
|
|
# memory: 512Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
# Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled
|
|
ROOK_ENFORCE_HOST_NETWORK: "false"
|
|
|
|
# RevisionHistoryLimit value for all deployments created by rook.
|
|
# ROOK_REVISION_HISTORY_LIMIT: "3"
|
|
|
|
# Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used
|
|
ROOK_CUSTOM_HOSTNAME_LABEL: ""
|
|
---
|
|
# CSI operator resources managed by the admin.
|
|
# These CRs define the configuration for CSI drivers deployed by the ceph-csi-operator.
|
|
# Customize these as needed for your environment.
|
|
|
|
# ImageSet ConfigMap defines the container images used by the CSI drivers.
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: rook-csi-operator-image-set-configmap
|
|
namespace: rook-ceph # namespace:operator
|
|
data:
|
|
provisioner: "registry.k8s.io/sig-storage/csi-provisioner:v6.1.1"
|
|
attacher: "registry.k8s.io/sig-storage/csi-attacher:v4.11.0"
|
|
resizer: "registry.k8s.io/sig-storage/csi-resizer:v2.1.0"
|
|
snapshotter: "registry.k8s.io/sig-storage/csi-snapshotter:v8.5.0"
|
|
registrar: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.16.0"
|
|
plugin: "quay.io/cephcsi/cephcsi:v3.16.2"
|
|
addons: "quay.io/csiaddons/k8s-sidecar:v0.14.0"
|
|
---
|
|
# OperatorConfig defines the default settings for all CSI drivers.
|
|
apiVersion: csi.ceph.io/v1
|
|
kind: OperatorConfig
|
|
metadata:
|
|
name: ceph-csi-operator-config
|
|
namespace: rook-ceph # namespace:operator
|
|
spec:
|
|
driverSpecDefaults:
|
|
log:
|
|
verbosity: 0
|
|
imageSet:
|
|
name: rook-csi-operator-image-set-configmap
|
|
enableMetadata: true
|
|
generateOMapInfo: false
|
|
fsGroupPolicy: File
|
|
deployCsiAddons: true
|
|
cephFsClientType: kernel
|
|
nodePlugin:
|
|
priorityClassName: "system-node-critical"
|
|
kubeletDirPath: /var/lib/kubelet
|
|
enableSeLinuxHostMount: false
|
|
controllerPlugin:
|
|
priorityClassName: "system-cluster-critical"
|
|
replicas: 1
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
---
|
|
# RBD CSI Driver
|
|
apiVersion: csi.ceph.io/v1
|
|
kind: Driver
|
|
metadata:
|
|
name: rook-ceph.rbd.csi.ceph.com # csi-provisioner-name
|
|
namespace: rook-ceph # namespace:operator
|
|
spec:
|
|
fsGroupPolicy: File
|
|
nodePlugin:
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
controllerPlugin: {}
|
|
---
|
|
# CephFS CSI Driver
|
|
apiVersion: csi.ceph.io/v1
|
|
kind: Driver
|
|
metadata:
|
|
name: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name
|
|
namespace: rook-ceph # namespace:operator
|
|
spec:
|
|
fsGroupPolicy: File
|
|
cephFsClientType: kernel
|
|
nodePlugin:
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
controllerPlugin: {}
|
|
---
|
|
# OLM: BEGIN OPERATOR DEPLOYMENT
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: rook-ceph-operator
|
|
namespace: rook-ceph # namespace:operator
|
|
labels:
|
|
operator: rook
|
|
storage-backend: ceph
|
|
app.kubernetes.io/name: rook-ceph
|
|
app.kubernetes.io/instance: rook-ceph
|
|
app.kubernetes.io/component: rook-ceph-operator
|
|
app.kubernetes.io/part-of: rook-ceph-operator
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: rook-ceph-operator
|
|
strategy:
|
|
type: Recreate
|
|
replicas: 1
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: rook-ceph-operator
|
|
spec:
|
|
tolerations:
|
|
- effect: NoExecute
|
|
key: node.kubernetes.io/unreachable
|
|
operator: Exists
|
|
tolerationSeconds: 5
|
|
serviceAccountName: rook-ceph-system
|
|
containers:
|
|
- name: rook-ceph-operator
|
|
image: docker.io/rook/ceph:master
|
|
args: ["ceph", "operator"]
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 2016
|
|
runAsGroup: 2016
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumeMounts:
|
|
- mountPath: /var/lib/rook
|
|
name: rook-config
|
|
- mountPath: /etc/ceph
|
|
name: default-config-dir
|
|
env:
|
|
# If the operator should only watch for cluster CRDs in the same namespace, set this to "true".
|
|
# If this is not set to true, the operator will watch for cluster CRDs in all namespaces.
|
|
- name: ROOK_CURRENT_NAMESPACE_ONLY
|
|
value: "false"
|
|
|
|
# The default is 1, which means each cluster is reconciled sequentially within the operator.
|
|
# When set to a value greater than 1, the operator can reconcile multiple clusters concurrently,
|
|
# and all logging will be interleaved in the same operator log.
|
|
# Changing this value requires restarting the operator pod.
|
|
- name: ROOK_RECONCILE_CONCURRENT_CLUSTERS
|
|
value: "1"
|
|
|
|
# Whether to start pods as privileged that mount a host path, which includes the Ceph mon, osd pods and csi provisioners(if logrotation is on).
|
|
# Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues.
|
|
# For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
|
|
- name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
|
|
value: "false"
|
|
# Whether to run Ceph mon pods as root (runAsUser: 0) at the pod level.
|
|
# Set this to true if mon pods are CrashLooping on SELinux-enabled systems despite other settings.
|
|
# For more context, see https://github.com/rook/rook/issues/15564
|
|
- name: ROOK_CEPH_MON_RUN_AS_ROOT
|
|
value: "false"
|
|
# Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+".
|
|
# In case of more than one regex, use comma to separate between them.
|
|
# Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
|
|
# Add regex expression after putting a comma to blacklist a disk
|
|
# If value is empty, the default regex will be used.
|
|
- name: DISCOVER_DAEMON_UDEV_BLACKLIST
|
|
value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
|
|
|
|
# Time to wait until the node controller will move Rook pods to other
|
|
# nodes after detecting an unreachable node.
|
|
# Pods affected by this setting are:
|
|
# mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox
|
|
# The value used in this variable replaces the default value of 300 secs
|
|
# added automatically by k8s as Toleration for
|
|
# <node.kubernetes.io/unreachable>
|
|
# The total amount of time to reschedule Rook pods in healthy nodes
|
|
# before detecting a <not ready node> condition will be the sum of:
|
|
# --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag)
|
|
# --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds
|
|
- name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS
|
|
value: "5"
|
|
|
|
# The name of the node to pass with the downward API
|
|
- name: NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
# The pod name to pass with the downward API
|
|
- name: POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
# The pod namespace to pass with the downward API
|
|
- name: POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
# Recommended resource requests and limits, if desired
|
|
#resources:
|
|
# limits:
|
|
# memory: 512Mi
|
|
# requests:
|
|
# cpu: 200m
|
|
# memory: 128Mi
|
|
|
|
# Uncomment it to run lib bucket provisioner in multithreaded mode
|
|
#- name: LIB_BUCKET_PROVISIONER_THREADS
|
|
# value: "5"
|
|
|
|
# Uncomment these two settings to run the operator on the host network
|
|
# hostNetwork: true
|
|
# dnsPolicy: ClusterFirstWithHostNet
|
|
volumes:
|
|
- name: rook-config
|
|
emptyDir: {}
|
|
- name: default-config-dir
|
|
emptyDir: {}
|
|
# OLM: END OPERATOR DEPLOYMENT
|