#################################################################################################################
# Define the settings for the rook-ceph cluster with common settings for a production cluster.
# All nodes with available raw devices will be used for the Ceph cluster. At least three nodes are required
# in this example. See the documentation for more details on storage settings available.

# For example, to create the cluster:
#   kubectl create -f crds.yaml -f common.yaml -f operator.yaml
#   kubectl create -f cluster-stretched.yaml
#################################################################################################################

apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph # namespace:cluster
spec:
  # If there are multiple clusters, the directory must be unique for each cluster.
  dataDirHostPath: /var/lib/rook
  mon:
    # Five mons must be created for stretch mode
    count: 5
    allowMultiplePerNode: false
    stretchCluster:
      # The ceph failure domain will be extracted from the label, which by default is the zone. The nodes running OSDs must have
      # this label in order for the OSDs to be configured in the correct topology. For topology labels, see
      # https://rook.io/docs/rook/latest/CRDs/ceph-cluster-crd/#osd-topology.
      failureDomainLabel: topology.kubernetes.io/zone
      # The sub failure domain is the secondary level at which the data will be placed to maintain data durability and availability.
      # The default is "host", which means that each OSD must be on a different node and you would need at least two nodes per zone.
      # If the subFailureDomain is set to "osd", the OSDs would be allowed anywhere in the same zone including on the same node.
      # If set to "rack" or some other intermediate failure domain, those labels would also need to be set on the nodes where
      # the osds are started.
      subFailureDomain: host
      zones:
        - name: a
          arbiter: true
        - name: b
        - name: c
  mgr:
    count: 2
  cephVersion:
    image: quay.io/ceph/ceph:v19.2.1
    allowUnsupported: true
  skipUpgradeChecks: false
  continueUpgradeAfterChecksEvenIfNotHealthy: false
  dashboard:
    enabled: true
    ssl: true
  storage:
    useAllNodes: true
    useAllDevices: true
    deviceFilter: ""
  # OSD placement is expected to include the non-arbiter zones
  placement:
    # The arbiter mon can have its own placement settings that will be different from the mons.
    # If the arbiter section is not included in the placement, the arbiter will use the same placement
    # settings as other mons. In this example, the arbiter has a toleration to run on a control-plane node.
    arbiter:
      tolerations:
        # kubernetes v1.24 clusters would need the taint `node-role.kubernetes.io/control-plane`
        # configuration. For earlier versions you may use `node-role.kubernetes.io/master` if
        # available in your cluster.
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
    osd:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: topology.kubernetes.io/zone
                  operator: In
                  values:
                    - b
                    - c
  priorityClassNames:
    mon: system-node-critical
    osd: system-node-critical
    mgr: system-cluster-critical
  disruptionManagement:
    managePodBudgets: true
---
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: builtin-mgr
  namespace: rook-ceph # namespace:cluster
spec:
  name: .mgr
  failureDomain: zone
  replicated:
    size: 4
    requireSafeReplicaSize: true
    replicasPerFailureDomain: 2
    subFailureDomain: host