Merge pull request #17949 from rook/mergify/bp/release-1.20/pr-17917

helm: add multisite CR support to rook-ceph-cluster chart (backport #17917)

Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md

@@ -126,6 +126,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject
 | `ingress.tls` | Ingress tls | `/` |
 | `ingress.ingressClassName` | Ingress tls | `""` |
 
+### **Ceph Object Multisite**
+
+The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define
+[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are
+commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a
+`cephObjectStores` entry at the zone via its `spec.zone.name`.
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` |
+| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set |
+| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` |
+| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml |
+| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` |
+| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml |
+
 ### **Existing Clusters**
 
 If you have an existing CephCluster CR that was created without the helm chart and you want the helm

Documentation/Helm-Charts/ceph-cluster-chart.md

@@ -163,6 +163,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject
 | `ingress.tls` | Ingress tls | `/` |
 | `ingress.ingressClassName` | Ingress tls | `""` |
 
+### **Ceph Object Multisite**
+
+The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define
+[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are
+commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a
+`cephObjectStores` entry at the zone via its `spec.zone.name`.
+
+| Parameter | Description | Default |
+| --------- | ----------- | ------- |
+| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` |
+| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set |
+| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` |
+| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml |
+| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` |
+| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml |
+
 ### **Existing Clusters**
 
 If you have an existing CephCluster CR that was created without the helm chart and you want the helm

deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml

@@ -0,0 +1,13 @@
+{{- $root := . -}}
+{{- range $realm := .Values.cephObjectRealms }}
+---
+kind: CephObjectRealm
+apiVersion: ceph.rook.io/v1
+metadata:
+  name: {{ $realm.name }}
+  namespace: {{ $root.Release.Namespace }} # namespace:cluster
+{{- with $realm.spec }}
+spec:
+  {{- . | toYaml | nindent 2 }}
+{{- end }}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml

@@ -0,0 +1,11 @@
+{{- $root := . -}}
+{{- range $zone := .Values.cephObjectZones }}
+---
+kind: CephObjectZone
+apiVersion: ceph.rook.io/v1
+metadata:
+  name: {{ $zone.name }}
+  namespace: {{ $root.Release.Namespace }} # namespace:cluster
+spec:
+  {{- $zone.spec | toYaml | nindent 2 }}
+{{- end }}

deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml

@@ -0,0 +1,11 @@
+{{- $root := . -}}
+{{- range $zonegroup := .Values.cephObjectZoneGroups }}
+---
+kind: CephObjectZoneGroup
+apiVersion: ceph.rook.io/v1
+metadata:
+  name: {{ $zonegroup.name }}
+  namespace: {{ $root.Release.Namespace }} # namespace:cluster
+spec:
+  {{- $zonegroup.spec | toYaml | nindent 2 }}
+{{- end }}

deploy/charts/rook-ceph-cluster/values.yaml

@@ -698,6 +698,43 @@ cephObjectStores:
       #   - name: internal
       #     namespace: kube-system
       #     sectionName: https
+## CephObjectRealm, CephObjectZoneGroup, and CephObjectZone configure RGW multisite. They are
+## disabled by default; remove the comments and set desired values to enable them. See
+## https://rook.io/docs/rook/latest/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/
+## The example below creates a new realm on this (primary) cluster. On a secondary cluster that
+## syncs from another realm, add a `spec.pull.endpoint` to the realm instead (see the realm CRD docs).
+#cephObjectRealms:
+#  - name: realm-a
+#    # spec is only required to pull a realm from another cluster; omit it to create a new realm
+#    # spec:
+#    #   pull:
+#    #     endpoint: http://<master-zone-endpoint>
+#cephObjectZoneGroups:
+#  - name: zonegroup-a
+#    spec:
+#      realm: realm-a
+#cephObjectZones:
+#  - name: zone-a
+#    spec:
+#      zoneGroup: zonegroup-a
+#      metadataPool:
+#        failureDomain: host
+#        replicated:
+#          size: 3
+#          requireSafeReplicaSize: true
+#      dataPool:
+#        failureDomain: host
+#        replicated:
+#          size: 3
+#          requireSafeReplicaSize: true
+#        parameters:
+#          compression_mode: none
+#          bulk: "true"
+#      # recommended to set this value if ingress used for exposing rgw endpoints
+#      # customEndpoints:
+#      #   - "http://rgw-a.fqdn"
+#      # if the pools need to be removed from backend enable following setting
+#      # preservePoolsOnDelete: false
 ## cephECBlockPools are disabled by default, please remove the comments and set desired values to enable it
 ## For erasure coded a replicated metadata pool is required.
 ## https://rook.io/docs/rook/latest/CRDs/Shared-Filesystem/ceph-filesystem-crd/#erasure-coded

tests/framework/installer/ceph_helm_installer.go

@@ -285,6 +285,49 @@ func (h *CephInstaller) removeCephClusterHelmResources() {
 	if !h.k8shelper.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", ObjectStoreName), h.settings.Namespace) {
 		assert.Fail(h.T(), "rgw did not stop via helm uninstall")
 	}
+
+	if h.settings.UseMultisiteObjectStore {
+		h.removeMultisiteHelmResources(ObjectStoreName)
+	}
+}
+
+// removeMultisiteHelmResources tears down the CephObjectRealm, CephObjectZoneGroup,
+// CephObjectZone, and shared rgw pools that CreateObjectStoreConfiguration added
+// to the chart. They are not part of the chart's default storage CRs, so they
+// must be removed explicitly or they orphan and trip the teardown pool check.
+// Order matters: the zone's deletion finalizer runs radosgw-admin against realm
+// metadata stored in the .rgw.root pool, so the store (already deleted above),
+// zone, zone group, and realm must all be gone before their pools are removed.
+func (h *CephInstaller) removeMultisiteHelmResources(name string) {
+	ctx := context.TODO()
+	ceph := h.k8shelper.RookClientset.CephV1()
+	ns := h.settings.Namespace
+
+	deleteAndWait := func(kind, crName string, del func() error, get func() error) {
+		if err := del(); err != nil {
+			assert.True(h.T(), kerrors.IsNotFound(err), "unexpected error deleting %s %q", kind, crName)
+			return
+		}
+		if err := h.k8shelper.WaitForCustomResourceDeletion(ns, crName, get); err != nil {
+			assert.NoError(h.T(), err, "failed waiting for %s %q deletion", kind, crName)
+		}
+	}
+
+	deleteAndWait("CephObjectZone", name,
+		func() error { return ceph.CephObjectZones(ns).Delete(ctx, name, v1.DeleteOptions{}) },
+		func() error { _, err := ceph.CephObjectZones(ns).Get(ctx, name, v1.GetOptions{}); return err })
+	deleteAndWait("CephObjectZoneGroup", name,
+		func() error { return ceph.CephObjectZoneGroups(ns).Delete(ctx, name, v1.DeleteOptions{}) },
+		func() error { _, err := ceph.CephObjectZoneGroups(ns).Get(ctx, name, v1.GetOptions{}); return err })
+	deleteAndWait("CephObjectRealm", name,
+		func() error { return ceph.CephObjectRealms(ns).Delete(ctx, name, v1.DeleteOptions{}) },
+		func() error { _, err := ceph.CephObjectRealms(ns).Get(ctx, name, v1.GetOptions{}); return err })
+
+	for poolName := range rgwSharedPools(name) {
+		deleteAndWait("CephBlockPool", poolName,
+			func() error { return ceph.CephBlockPools(ns).Delete(ctx, poolName, v1.DeleteOptions{}) },
+			func() error { _, err := ceph.CephBlockPools(ns).Get(ctx, poolName, v1.GetOptions{}); return err })
+	}
 }
 
 // ConfirmHelmClusterInstalledCorrectly runs some validation to check whether the helm chart installed correctly.
@@ -381,8 +424,32 @@ func (h *CephInstaller) CreateFileSystemConfiguration(values map[string]interfac
 	return nil
 }
 
-// CreateObjectStoreConfiguration creates an object store configuration
+// rgwSharedPools maps the CephBlockPool CR name to its rados pool name (empty
+// means the operator derives the pool name from the CR name) for the multisite
+// object store configured by CreateObjectStoreConfiguration. .rgw.root is the
+// realm-global metadata pool; the operator sets pg_num_min=8 on it, so it must
+// be created with pg_num>=8 or the zone reconcile fails.
+func rgwSharedPools(name string) map[string]string {
+	return map[string]string{
+		"rgw.root":                     ".rgw.root",
+		name + ".rgw.control":          "",
+		name + ".rgw.meta":             "",
+		name + ".rgw.log":              "",
+		name + ".rgw.otp":              "",
+		name + ".rgw.buckets.index":    "",
+		name + ".rgw.buckets.data":     "",
+		name + ".rgw.buckets.data.foo": "",
+	}
+}
+
+// CreateObjectStoreConfiguration adds the chart's object store and its bucket
+// storage class to the Helm values. When UseMultisiteObjectStore is set it
+// configures an RGW multisite zone; otherwise it configures a plain store.
 func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interface{}, name, scName string) error {
+	if h.settings.UseMultisiteObjectStore {
+		return h.createMultisiteObjectStoreConfiguration(values, name, scName)
+	}
+
 	testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false, false))
 	var testObjectStoreCRD map[string]interface{}
 	if err := yaml.Unmarshal(testObjectStoreBytes, &testObjectStoreCRD); err != nil {
@@ -409,3 +476,112 @@ func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interfa
 	}
 	return nil
 }
+
+// createMultisiteObjectStoreConfiguration configures the object store as an RGW
+// multisite zone, exercising the chart's CephObjectRealm, CephObjectZoneGroup,
+// and CephObjectZone templates on install. The realm/zone group/zone/shared-pools
+// layout mirrors the fixture in tests/integration/object/util/sharedstore.
+func (h *CephInstaller) createMultisiteObjectStoreConfiguration(values map[string]interface{}, name, scName string) error {
+	poolNames := rgwSharedPools(name)
+	rgwPools := make([]map[string]interface{}, 0, len(poolNames))
+	for poolName, radosName := range poolNames {
+		pgNum := "1"
+		if radosName == ".rgw.root" {
+			pgNum = "8"
+		}
+		spec := map[string]interface{}{
+			"replicated": map[string]interface{}{
+				"size":                   1,
+				"requireSafeReplicaSize": false,
+			},
+			"parameters": map[string]interface{}{
+				"pg_autoscale_mode": "off",
+				"pg_num":            pgNum,
+			},
+		}
+		// spec.name is enum-validated (.rgw.root/.nfs/.mgr) and must be omitted
+		// rather than set to "" when the pool name matches the CR name.
+		if radosName != "" {
+			spec["name"] = radosName
+		}
+		rgwPools = append(rgwPools, map[string]interface{}{
+			"name": poolName,
+			"spec": spec,
+			"storageClass": map[string]interface{}{
+				"enabled": false,
+			},
+		})
+	}
+
+	// Append the rgw pools to the block pools already configured for CSI so both
+	// sets land in the chart's cephBlockPools list.
+	existingPools, _ := values["cephBlockPools"].([]map[string]interface{})
+	values["cephBlockPools"] = append(existingPools, rgwPools...)
+
+	values["cephObjectRealms"] = []map[string]interface{}{
+		{
+			"name": name,
+		},
+	}
+	values["cephObjectZoneGroups"] = []map[string]interface{}{
+		{
+			"name": name,
+			"spec": map[string]interface{}{
+				"realm": name,
+			},
+		},
+	}
+	values["cephObjectZones"] = []map[string]interface{}{
+		{
+			"name": name,
+			"spec": map[string]interface{}{
+				"zoneGroup": name,
+				"sharedPools": map[string]interface{}{
+					"poolPlacements": []map[string]interface{}{
+						{
+							"name":             "default",
+							"default":          true,
+							"metadataPoolName": name + ".rgw.buckets.index",
+							"dataPoolName":     name + ".rgw.buckets.data",
+							"storageClasses": []map[string]interface{}{
+								{
+									"name":         "FOO",
+									"dataPoolName": name + ".rgw.buckets.data.foo",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	storageClassBytes := []byte(h.Manifests.GetBucketStorageClass(name, scName, "Delete"))
+	var testObjectStoreSC map[string]interface{}
+	if err := yaml.Unmarshal(storageClassBytes, &testObjectStoreSC); err != nil {
+		return err
+	}
+
+	values["cephObjectStores"] = []map[string]interface{}{
+		{
+			"name": name,
+			"spec": map[string]interface{}{
+				"zone": map[string]interface{}{
+					"name": name,
+				},
+				"gateway": map[string]interface{}{
+					"port":      8080,
+					"instances": 2,
+					"resources": nil,
+				},
+			},
+			"storageClass": map[string]interface{}{
+				"enabled":       true,
+				"name":          scName,
+				"parameters":    testObjectStoreSC["parameters"],
+				"reclaimPolicy": "Delete",
+			},
+		},
+	}
+	return nil
+}

tests/framework/installer/ceph_settings.go

@@ -54,6 +54,12 @@ type TestCephSettings struct {
 	KubernetesVersion           string
 	EnableCsiOperator           bool
 	ClusterConcurrency          int
+
+	// UseMultisiteObjectStore configures the Helm chart's object store as an RGW
+	// multisite zone (realm/zone group/zone/shared pools) instead of a plain
+	// store. Only valid for local-chart installs, which carry the multisite
+	// templates, so it must stay off for the upgrade suite's older base chart.
+	UseMultisiteObjectStore bool
 }
 
 func (s *TestCephSettings) ApplyEnvVars() {

tests/integration/ceph_helm_test.go

@@ -59,19 +59,20 @@ type HelmSuite struct {
 func (h *HelmSuite) SetupSuite() {
 	namespace := "helm-ns"
 	h.settings = &installer.TestCephSettings{
-		Namespace:            namespace,
-		OperatorNamespace:    namespace,
-		StorageClassName:     "",
-		UseHelm:              true,
-		UsePVC:               false,
-		Mons:                 1,
-		SkipOSDCreation:      false,
-		EnableDiscovery:      true,
-		ChangeHostName:       true,
-		ConnectionsEncrypted: true,
-		RookVersion:          installer.LocalBuildTag,
-		CephVersion:          installer.SquidVersion,
-		EnableCsiOperator:    true,
+		Namespace:               namespace,
+		OperatorNamespace:       namespace,
+		StorageClassName:        "",
+		UseHelm:                 true,
+		UseMultisiteObjectStore: true,
+		UsePVC:                  false,
+		Mons:                    1,
+		SkipOSDCreation:         false,
+		EnableDiscovery:         true,
+		ChangeHostName:          true,
+		ConnectionsEncrypted:    true,
+		RookVersion:             installer.LocalBuildTag,
+		CephVersion:             installer.SquidVersion,
+		EnableCsiOperator:       true,
 	}
 	h.settings.ApplyEnvVars()
 	h.installer, h.k8shelper = StartTestCluster(h.T, h.settings)