From 2fe4b8f354d5106f0053911f287f36ba6c1b9cb1 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 8 Jul 2026 14:18:25 -0700 Subject: [PATCH 1/2] helm: add multisite CR support to rook-ceph-cluster chart Add optional CephObjectRealm, CephObjectZoneGroup, and CephObjectZone resources to the rook-ceph-cluster chart so an RGW multisite topology can be deployed from the chart alongside CephObjectStore. Each resource type follows the existing per-list template idiom: a new template ranges over a values list and renders name, namespace, and spec. The realm template renders spec only when set, supporting both a new realm (no spec) and a pulled realm (spec.pull.endpoint). The three lists ship commented out in values.yaml with a multisite example, matching the opt-in cephECBlockPools precedent, so a default install creates no multisite resources. Signed-off-by: Joshua Hoblitt (cherry picked from commit 8989bff6533634f7a2019e881116b332936d5f64) --- .../Helm-Charts/ceph-cluster-chart.gotmpl.md | 16 ++++++++ .../Helm-Charts/ceph-cluster-chart.md | 16 ++++++++ .../templates/cephobjectrealm.yaml | 13 +++++++ .../templates/cephobjectzone.yaml | 11 ++++++ .../templates/cephobjectzonegroup.yaml | 11 ++++++ deploy/charts/rook-ceph-cluster/values.yaml | 37 +++++++++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml create mode 100644 deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml create mode 100644 deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md index 293e5ccf3..7853aa308 100644 --- a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md @@ -126,6 +126,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject | `ingress.tls` | Ingress tls | `/` | | `ingress.ingressClassName` | Ingress tls | `""` | +### **Ceph Object Multisite** + +The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define +[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are +commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a +`cephObjectStores` entry at the zone via its `spec.zone.name`. + +| Parameter | Description | Default | +| --------- | ----------- | ------- | +| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` | +| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set | +| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` | +| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml | +| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` | +| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml | + ### **Existing Clusters** If you have an existing CephCluster CR that was created without the helm chart and you want the helm diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.md b/Documentation/Helm-Charts/ceph-cluster-chart.md index d4efcb282..57766a1b1 100644 --- a/Documentation/Helm-Charts/ceph-cluster-chart.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.md @@ -163,6 +163,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject | `ingress.tls` | Ingress tls | `/` | | `ingress.ingressClassName` | Ingress tls | `""` | +### **Ceph Object Multisite** + +The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define +[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are +commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a +`cephObjectStores` entry at the zone via its `spec.zone.name`. + +| Parameter | Description | Default | +| --------- | ----------- | ------- | +| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` | +| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set | +| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` | +| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml | +| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` | +| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml | + ### **Existing Clusters** If you have an existing CephCluster CR that was created without the helm chart and you want the helm diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml new file mode 100644 index 000000000..639f0e9b5 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml @@ -0,0 +1,13 @@ +{{- $root := . -}} +{{- range $realm := .Values.cephObjectRealms }} +--- +kind: CephObjectRealm +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $realm.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +{{- with $realm.spec }} +spec: + {{- . | toYaml | nindent 2 }} +{{- end }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml new file mode 100644 index 000000000..52639eaf5 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml @@ -0,0 +1,11 @@ +{{- $root := . -}} +{{- range $zone := .Values.cephObjectZones }} +--- +kind: CephObjectZone +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $zone.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +spec: + {{- $zone.spec | toYaml | nindent 2 }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml new file mode 100644 index 000000000..cec226f48 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml @@ -0,0 +1,11 @@ +{{- $root := . -}} +{{- range $zonegroup := .Values.cephObjectZoneGroups }} +--- +kind: CephObjectZoneGroup +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $zonegroup.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +spec: + {{- $zonegroup.spec | toYaml | nindent 2 }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index d3f530512..7785748e1 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -698,6 +698,43 @@ cephObjectStores: # - name: internal # namespace: kube-system # sectionName: https +## CephObjectRealm, CephObjectZoneGroup, and CephObjectZone configure RGW multisite. They are +## disabled by default; remove the comments and set desired values to enable them. See +## https://rook.io/docs/rook/latest/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/ +## The example below creates a new realm on this (primary) cluster. On a secondary cluster that +## syncs from another realm, add a `spec.pull.endpoint` to the realm instead (see the realm CRD docs). +#cephObjectRealms: +# - name: realm-a +# # spec is only required to pull a realm from another cluster; omit it to create a new realm +# # spec: +# # pull: +# # endpoint: http:// +#cephObjectZoneGroups: +# - name: zonegroup-a +# spec: +# realm: realm-a +#cephObjectZones: +# - name: zone-a +# spec: +# zoneGroup: zonegroup-a +# metadataPool: +# failureDomain: host +# replicated: +# size: 3 +# requireSafeReplicaSize: true +# dataPool: +# failureDomain: host +# replicated: +# size: 3 +# requireSafeReplicaSize: true +# parameters: +# compression_mode: none +# bulk: "true" +# # recommended to set this value if ingress used for exposing rgw endpoints +# # customEndpoints: +# # - "http://rgw-a.fqdn" +# # if the pools need to be removed from backend enable following setting +# # preservePoolsOnDelete: false ## cephECBlockPools are disabled by default, please remove the comments and set desired values to enable it ## For erasure coded a replicated metadata pool is required. ## https://rook.io/docs/rook/latest/CRDs/Shared-Filesystem/ceph-filesystem-crd/#erasure-coded From d82cfb9d4e2e991aaca80846f116da00051b9604 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 8 Jul 2026 15:00:47 -0700 Subject: [PATCH 2/2] test: exercise multisite chart values in CephHelmSuite Configure the CephHelmSuite object store as an RGW multisite zone so the chart's CephObjectRealm, CephObjectZoneGroup, and CephObjectZone templates are exercised on a Helm install. The operator must reconcile the realm, zone group, zone, and store to Ready, which the existing install check already gates on via the RGW pod count. The realm/zone group/zone and shared-pool layout mirror the fixture in tests/integration/object/util/sharedstore: a zone backed by explicit CephBlockPools referenced through sharedPools.poolPlacements. The rgw pools are appended to the block pools already configured for CSI, and are torn down along with the multisite CRs when the default storage CRs are removed. Gate the multisite configuration behind a new UseMultisiteObjectStore setting enabled only for CephHelmSuite. The other Helm-based suites keep the plain object store; the upgrade suite in particular installs an older base chart that lacks the multisite templates and cannot resolve a zone-based store. Signed-off-by: Joshua Hoblitt (cherry picked from commit 8a72d801278b84788129cf003687d1f1276046ef) # Conflicts: # tests/integration/ceph_helm_test.go --- .../installer/ceph_helm_installer.go | 178 +++++++++++++++++- tests/framework/installer/ceph_settings.go | 6 + tests/integration/ceph_helm_test.go | 27 +-- 3 files changed, 197 insertions(+), 14 deletions(-) diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 6be6bb6d0..051241a1e 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -285,6 +285,49 @@ func (h *CephInstaller) removeCephClusterHelmResources() { if !h.k8shelper.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", ObjectStoreName), h.settings.Namespace) { assert.Fail(h.T(), "rgw did not stop via helm uninstall") } + + if h.settings.UseMultisiteObjectStore { + h.removeMultisiteHelmResources(ObjectStoreName) + } +} + +// removeMultisiteHelmResources tears down the CephObjectRealm, CephObjectZoneGroup, +// CephObjectZone, and shared rgw pools that CreateObjectStoreConfiguration added +// to the chart. They are not part of the chart's default storage CRs, so they +// must be removed explicitly or they orphan and trip the teardown pool check. +// Order matters: the zone's deletion finalizer runs radosgw-admin against realm +// metadata stored in the .rgw.root pool, so the store (already deleted above), +// zone, zone group, and realm must all be gone before their pools are removed. +func (h *CephInstaller) removeMultisiteHelmResources(name string) { + ctx := context.TODO() + ceph := h.k8shelper.RookClientset.CephV1() + ns := h.settings.Namespace + + deleteAndWait := func(kind, crName string, del func() error, get func() error) { + if err := del(); err != nil { + assert.True(h.T(), kerrors.IsNotFound(err), "unexpected error deleting %s %q", kind, crName) + return + } + if err := h.k8shelper.WaitForCustomResourceDeletion(ns, crName, get); err != nil { + assert.NoError(h.T(), err, "failed waiting for %s %q deletion", kind, crName) + } + } + + deleteAndWait("CephObjectZone", name, + func() error { return ceph.CephObjectZones(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZones(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectZoneGroup", name, + func() error { return ceph.CephObjectZoneGroups(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZoneGroups(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectRealm", name, + func() error { return ceph.CephObjectRealms(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectRealms(ns).Get(ctx, name, v1.GetOptions{}); return err }) + + for poolName := range rgwSharedPools(name) { + deleteAndWait("CephBlockPool", poolName, + func() error { return ceph.CephBlockPools(ns).Delete(ctx, poolName, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephBlockPools(ns).Get(ctx, poolName, v1.GetOptions{}); return err }) + } } // ConfirmHelmClusterInstalledCorrectly runs some validation to check whether the helm chart installed correctly. @@ -381,8 +424,32 @@ func (h *CephInstaller) CreateFileSystemConfiguration(values map[string]interfac return nil } -// CreateObjectStoreConfiguration creates an object store configuration +// rgwSharedPools maps the CephBlockPool CR name to its rados pool name (empty +// means the operator derives the pool name from the CR name) for the multisite +// object store configured by CreateObjectStoreConfiguration. .rgw.root is the +// realm-global metadata pool; the operator sets pg_num_min=8 on it, so it must +// be created with pg_num>=8 or the zone reconcile fails. +func rgwSharedPools(name string) map[string]string { + return map[string]string{ + "rgw.root": ".rgw.root", + name + ".rgw.control": "", + name + ".rgw.meta": "", + name + ".rgw.log": "", + name + ".rgw.otp": "", + name + ".rgw.buckets.index": "", + name + ".rgw.buckets.data": "", + name + ".rgw.buckets.data.foo": "", + } +} + +// CreateObjectStoreConfiguration adds the chart's object store and its bucket +// storage class to the Helm values. When UseMultisiteObjectStore is set it +// configures an RGW multisite zone; otherwise it configures a plain store. func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + if h.settings.UseMultisiteObjectStore { + return h.createMultisiteObjectStoreConfiguration(values, name, scName) + } + testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false, false)) var testObjectStoreCRD map[string]interface{} if err := yaml.Unmarshal(testObjectStoreBytes, &testObjectStoreCRD); err != nil { @@ -409,3 +476,112 @@ func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interfa } return nil } + +// createMultisiteObjectStoreConfiguration configures the object store as an RGW +// multisite zone, exercising the chart's CephObjectRealm, CephObjectZoneGroup, +// and CephObjectZone templates on install. The realm/zone group/zone/shared-pools +// layout mirrors the fixture in tests/integration/object/util/sharedstore. +func (h *CephInstaller) createMultisiteObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + poolNames := rgwSharedPools(name) + rgwPools := make([]map[string]interface{}, 0, len(poolNames)) + for poolName, radosName := range poolNames { + pgNum := "1" + if radosName == ".rgw.root" { + pgNum = "8" + } + spec := map[string]interface{}{ + "replicated": map[string]interface{}{ + "size": 1, + "requireSafeReplicaSize": false, + }, + "parameters": map[string]interface{}{ + "pg_autoscale_mode": "off", + "pg_num": pgNum, + }, + } + // spec.name is enum-validated (.rgw.root/.nfs/.mgr) and must be omitted + // rather than set to "" when the pool name matches the CR name. + if radosName != "" { + spec["name"] = radosName + } + rgwPools = append(rgwPools, map[string]interface{}{ + "name": poolName, + "spec": spec, + "storageClass": map[string]interface{}{ + "enabled": false, + }, + }) + } + + // Append the rgw pools to the block pools already configured for CSI so both + // sets land in the chart's cephBlockPools list. + existingPools, _ := values["cephBlockPools"].([]map[string]interface{}) + values["cephBlockPools"] = append(existingPools, rgwPools...) + + values["cephObjectRealms"] = []map[string]interface{}{ + { + "name": name, + }, + } + values["cephObjectZoneGroups"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "realm": name, + }, + }, + } + values["cephObjectZones"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zoneGroup": name, + "sharedPools": map[string]interface{}{ + "poolPlacements": []map[string]interface{}{ + { + "name": "default", + "default": true, + "metadataPoolName": name + ".rgw.buckets.index", + "dataPoolName": name + ".rgw.buckets.data", + "storageClasses": []map[string]interface{}{ + { + "name": "FOO", + "dataPoolName": name + ".rgw.buckets.data.foo", + }, + }, + }, + }, + }, + }, + }, + } + + storageClassBytes := []byte(h.Manifests.GetBucketStorageClass(name, scName, "Delete")) + var testObjectStoreSC map[string]interface{} + if err := yaml.Unmarshal(storageClassBytes, &testObjectStoreSC); err != nil { + return err + } + + values["cephObjectStores"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zone": map[string]interface{}{ + "name": name, + }, + "gateway": map[string]interface{}{ + "port": 8080, + "instances": 2, + "resources": nil, + }, + }, + "storageClass": map[string]interface{}{ + "enabled": true, + "name": scName, + "parameters": testObjectStoreSC["parameters"], + "reclaimPolicy": "Delete", + }, + }, + } + return nil +} diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 5383f557b..090c542f7 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -54,6 +54,12 @@ type TestCephSettings struct { KubernetesVersion string EnableCsiOperator bool ClusterConcurrency int + + // UseMultisiteObjectStore configures the Helm chart's object store as an RGW + // multisite zone (realm/zone group/zone/shared pools) instead of a plain + // store. Only valid for local-chart installs, which carry the multisite + // templates, so it must stay off for the upgrade suite's older base chart. + UseMultisiteObjectStore bool } func (s *TestCephSettings) ApplyEnvVars() { diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 360368fec..4a9a342bc 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -59,19 +59,20 @@ type HelmSuite struct { func (h *HelmSuite) SetupSuite() { namespace := "helm-ns" h.settings = &installer.TestCephSettings{ - Namespace: namespace, - OperatorNamespace: namespace, - StorageClassName: "", - UseHelm: true, - UsePVC: false, - Mons: 1, - SkipOSDCreation: false, - EnableDiscovery: true, - ChangeHostName: true, - ConnectionsEncrypted: true, - RookVersion: installer.LocalBuildTag, - CephVersion: installer.SquidVersion, - EnableCsiOperator: true, + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UseMultisiteObjectStore: true, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + EnableDiscovery: true, + ChangeHostName: true, + ConnectionsEncrypted: true, + RookVersion: installer.LocalBuildTag, + CephVersion: installer.SquidVersion, + EnableCsiOperator: true, } h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings)