From d82cfb9d4e2e991aaca80846f116da00051b9604 Mon Sep 17 00:00:00 2001 From: Joshua Hoblitt Date: Wed, 8 Jul 2026 15:00:47 -0700 Subject: [PATCH] test: exercise multisite chart values in CephHelmSuite Configure the CephHelmSuite object store as an RGW multisite zone so the chart's CephObjectRealm, CephObjectZoneGroup, and CephObjectZone templates are exercised on a Helm install. The operator must reconcile the realm, zone group, zone, and store to Ready, which the existing install check already gates on via the RGW pod count. The realm/zone group/zone and shared-pool layout mirror the fixture in tests/integration/object/util/sharedstore: a zone backed by explicit CephBlockPools referenced through sharedPools.poolPlacements. The rgw pools are appended to the block pools already configured for CSI, and are torn down along with the multisite CRs when the default storage CRs are removed. Gate the multisite configuration behind a new UseMultisiteObjectStore setting enabled only for CephHelmSuite. The other Helm-based suites keep the plain object store; the upgrade suite in particular installs an older base chart that lacks the multisite templates and cannot resolve a zone-based store. Signed-off-by: Joshua Hoblitt (cherry picked from commit 8a72d801278b84788129cf003687d1f1276046ef) # Conflicts: # tests/integration/ceph_helm_test.go --- .../installer/ceph_helm_installer.go | 178 +++++++++++++++++- tests/framework/installer/ceph_settings.go | 6 + tests/integration/ceph_helm_test.go | 27 +-- 3 files changed, 197 insertions(+), 14 deletions(-) diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 6be6bb6d0..051241a1e 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -285,6 +285,49 @@ func (h *CephInstaller) removeCephClusterHelmResources() { if !h.k8shelper.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", ObjectStoreName), h.settings.Namespace) { assert.Fail(h.T(), "rgw did not stop via helm uninstall") } + + if h.settings.UseMultisiteObjectStore { + h.removeMultisiteHelmResources(ObjectStoreName) + } +} + +// removeMultisiteHelmResources tears down the CephObjectRealm, CephObjectZoneGroup, +// CephObjectZone, and shared rgw pools that CreateObjectStoreConfiguration added +// to the chart. They are not part of the chart's default storage CRs, so they +// must be removed explicitly or they orphan and trip the teardown pool check. +// Order matters: the zone's deletion finalizer runs radosgw-admin against realm +// metadata stored in the .rgw.root pool, so the store (already deleted above), +// zone, zone group, and realm must all be gone before their pools are removed. +func (h *CephInstaller) removeMultisiteHelmResources(name string) { + ctx := context.TODO() + ceph := h.k8shelper.RookClientset.CephV1() + ns := h.settings.Namespace + + deleteAndWait := func(kind, crName string, del func() error, get func() error) { + if err := del(); err != nil { + assert.True(h.T(), kerrors.IsNotFound(err), "unexpected error deleting %s %q", kind, crName) + return + } + if err := h.k8shelper.WaitForCustomResourceDeletion(ns, crName, get); err != nil { + assert.NoError(h.T(), err, "failed waiting for %s %q deletion", kind, crName) + } + } + + deleteAndWait("CephObjectZone", name, + func() error { return ceph.CephObjectZones(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZones(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectZoneGroup", name, + func() error { return ceph.CephObjectZoneGroups(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZoneGroups(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectRealm", name, + func() error { return ceph.CephObjectRealms(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectRealms(ns).Get(ctx, name, v1.GetOptions{}); return err }) + + for poolName := range rgwSharedPools(name) { + deleteAndWait("CephBlockPool", poolName, + func() error { return ceph.CephBlockPools(ns).Delete(ctx, poolName, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephBlockPools(ns).Get(ctx, poolName, v1.GetOptions{}); return err }) + } } // ConfirmHelmClusterInstalledCorrectly runs some validation to check whether the helm chart installed correctly. @@ -381,8 +424,32 @@ func (h *CephInstaller) CreateFileSystemConfiguration(values map[string]interfac return nil } -// CreateObjectStoreConfiguration creates an object store configuration +// rgwSharedPools maps the CephBlockPool CR name to its rados pool name (empty +// means the operator derives the pool name from the CR name) for the multisite +// object store configured by CreateObjectStoreConfiguration. .rgw.root is the +// realm-global metadata pool; the operator sets pg_num_min=8 on it, so it must +// be created with pg_num>=8 or the zone reconcile fails. +func rgwSharedPools(name string) map[string]string { + return map[string]string{ + "rgw.root": ".rgw.root", + name + ".rgw.control": "", + name + ".rgw.meta": "", + name + ".rgw.log": "", + name + ".rgw.otp": "", + name + ".rgw.buckets.index": "", + name + ".rgw.buckets.data": "", + name + ".rgw.buckets.data.foo": "", + } +} + +// CreateObjectStoreConfiguration adds the chart's object store and its bucket +// storage class to the Helm values. When UseMultisiteObjectStore is set it +// configures an RGW multisite zone; otherwise it configures a plain store. func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + if h.settings.UseMultisiteObjectStore { + return h.createMultisiteObjectStoreConfiguration(values, name, scName) + } + testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false, false)) var testObjectStoreCRD map[string]interface{} if err := yaml.Unmarshal(testObjectStoreBytes, &testObjectStoreCRD); err != nil { @@ -409,3 +476,112 @@ func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interfa } return nil } + +// createMultisiteObjectStoreConfiguration configures the object store as an RGW +// multisite zone, exercising the chart's CephObjectRealm, CephObjectZoneGroup, +// and CephObjectZone templates on install. The realm/zone group/zone/shared-pools +// layout mirrors the fixture in tests/integration/object/util/sharedstore. +func (h *CephInstaller) createMultisiteObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + poolNames := rgwSharedPools(name) + rgwPools := make([]map[string]interface{}, 0, len(poolNames)) + for poolName, radosName := range poolNames { + pgNum := "1" + if radosName == ".rgw.root" { + pgNum = "8" + } + spec := map[string]interface{}{ + "replicated": map[string]interface{}{ + "size": 1, + "requireSafeReplicaSize": false, + }, + "parameters": map[string]interface{}{ + "pg_autoscale_mode": "off", + "pg_num": pgNum, + }, + } + // spec.name is enum-validated (.rgw.root/.nfs/.mgr) and must be omitted + // rather than set to "" when the pool name matches the CR name. + if radosName != "" { + spec["name"] = radosName + } + rgwPools = append(rgwPools, map[string]interface{}{ + "name": poolName, + "spec": spec, + "storageClass": map[string]interface{}{ + "enabled": false, + }, + }) + } + + // Append the rgw pools to the block pools already configured for CSI so both + // sets land in the chart's cephBlockPools list. + existingPools, _ := values["cephBlockPools"].([]map[string]interface{}) + values["cephBlockPools"] = append(existingPools, rgwPools...) + + values["cephObjectRealms"] = []map[string]interface{}{ + { + "name": name, + }, + } + values["cephObjectZoneGroups"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "realm": name, + }, + }, + } + values["cephObjectZones"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zoneGroup": name, + "sharedPools": map[string]interface{}{ + "poolPlacements": []map[string]interface{}{ + { + "name": "default", + "default": true, + "metadataPoolName": name + ".rgw.buckets.index", + "dataPoolName": name + ".rgw.buckets.data", + "storageClasses": []map[string]interface{}{ + { + "name": "FOO", + "dataPoolName": name + ".rgw.buckets.data.foo", + }, + }, + }, + }, + }, + }, + }, + } + + storageClassBytes := []byte(h.Manifests.GetBucketStorageClass(name, scName, "Delete")) + var testObjectStoreSC map[string]interface{} + if err := yaml.Unmarshal(storageClassBytes, &testObjectStoreSC); err != nil { + return err + } + + values["cephObjectStores"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zone": map[string]interface{}{ + "name": name, + }, + "gateway": map[string]interface{}{ + "port": 8080, + "instances": 2, + "resources": nil, + }, + }, + "storageClass": map[string]interface{}{ + "enabled": true, + "name": scName, + "parameters": testObjectStoreSC["parameters"], + "reclaimPolicy": "Delete", + }, + }, + } + return nil +} diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 5383f557b..090c542f7 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -54,6 +54,12 @@ type TestCephSettings struct { KubernetesVersion string EnableCsiOperator bool ClusterConcurrency int + + // UseMultisiteObjectStore configures the Helm chart's object store as an RGW + // multisite zone (realm/zone group/zone/shared pools) instead of a plain + // store. Only valid for local-chart installs, which carry the multisite + // templates, so it must stay off for the upgrade suite's older base chart. + UseMultisiteObjectStore bool } func (s *TestCephSettings) ApplyEnvVars() { diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 360368fec..4a9a342bc 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -59,19 +59,20 @@ type HelmSuite struct { func (h *HelmSuite) SetupSuite() { namespace := "helm-ns" h.settings = &installer.TestCephSettings{ - Namespace: namespace, - OperatorNamespace: namespace, - StorageClassName: "", - UseHelm: true, - UsePVC: false, - Mons: 1, - SkipOSDCreation: false, - EnableDiscovery: true, - ChangeHostName: true, - ConnectionsEncrypted: true, - RookVersion: installer.LocalBuildTag, - CephVersion: installer.SquidVersion, - EnableCsiOperator: true, + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UseMultisiteObjectStore: true, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + EnableDiscovery: true, + ChangeHostName: true, + ConnectionsEncrypted: true, + RookVersion: installer.LocalBuildTag, + CephVersion: installer.SquidVersion, + EnableCsiOperator: true, } h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings)