From b43d45602bb0c0a420ee32903f2cfa9772267f26 Mon Sep 17 00:00:00 2001
From: travisn <tnielsen@redhat.com>
Date: Mon, 12 Jun 2023 12:28:10 -0600
Subject: [PATCH] pool: add .mgr pool to the stretch cluster examples

The stretch clusters must create pools with a pool spec
that is valid for the stretch scenario, with 4 replicas
and failure domain and sub failure domain as needed
to match the topology. Otherwise, ceph will create a
default .mgr pool which will have replica 3 which is
invalid for the stretch scenario.

Signed-off-by: travisn <tnielsen@redhat.com>
(cherry picked from commit 11d83bfbec493a653dafd5789d9ee8139fb4e5e7)
---
 deploy/examples/cluster-stretched-aws.yaml | 20 +++++++++++++++++---
 deploy/examples/cluster-stretched.yaml     | 14 ++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml
index b3f1e3707..f70164967 100644
--- a/deploy/examples/cluster-stretched-aws.yaml
+++ b/deploy/examples/cluster-stretched-aws.yaml
@@ -25,11 +25,11 @@ spec:
       # https://rook.io/docs/rook/latest/CRDs/ceph-cluster-crd/#osd-topology.
       failureDomainLabel: topology.kubernetes.io/zone
       # The sub failure domain is the secondary level at which the data will be placed to maintain data durability and availability.
-      # The default is "host", which means that each OSD must be on a different node and you would need at least two nodes per zone.
-      # If the subFailureDomain is set to "osd", the OSDs would be allowed anywhere in the same zone including on the same node.
+      # Since OSDs are portable, the default is "osd", which means that you would need at least two nodes per zone.
+      # If the subFailureDomain is set to "host", the OSDs should not be portable.
       # If set to "rack" or some other intermediate failure domain, those labels would also need to be set on the nodes where
       # the osds are started.
-      subFailureDomain: host
+      subFailureDomain: osd
       zones:
         - name: us-east-2a
           arbiter: true
@@ -141,3 +141,17 @@ spec:
     mgr: system-cluster-critical
   disruptionManagement:
     managePodBudgets: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: builtin-mgr
+  namespace: rook-ceph # namespace:cluster
+spec:
+  name: .mgr
+  failureDomain: zone
+  replicated:
+    size: 4
+    requireSafeReplicaSize: true
+    replicasPerFailureDomain: 2
+    subFailureDomain: osd
diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml
index e31068e54..f14c4c40d 100644
--- a/deploy/examples/cluster-stretched.yaml
+++ b/deploy/examples/cluster-stretched.yaml
@@ -79,3 +79,17 @@ spec:
     mgr: system-cluster-critical
   disruptionManagement:
     managePodBudgets: true
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: builtin-mgr
+  namespace: rook-ceph # namespace:cluster
+spec:
+  name: .mgr
+  failureDomain: zone
+  replicated:
+    size: 4
+    requireSafeReplicaSize: true
+    replicasPerFailureDomain: 2
+    subFailureDomain: host