diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md index 293e5ccf3..7853aa308 100644 --- a/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.gotmpl.md @@ -126,6 +126,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject | `ingress.tls` | Ingress tls | `/` | | `ingress.ingressClassName` | Ingress tls | `""` | +### **Ceph Object Multisite** + +The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define +[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are +commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a +`cephObjectStores` entry at the zone via its `spec.zone.name`. + +| Parameter | Description | Default | +| --------- | ----------- | ------- | +| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` | +| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set | +| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` | +| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml | +| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` | +| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml | + ### **Existing Clusters** If you have an existing CephCluster CR that was created without the helm chart and you want the helm diff --git a/Documentation/Helm-Charts/ceph-cluster-chart.md b/Documentation/Helm-Charts/ceph-cluster-chart.md index d4efcb282..57766a1b1 100644 --- a/Documentation/Helm-Charts/ceph-cluster-chart.md +++ b/Documentation/Helm-Charts/ceph-cluster-chart.md @@ -163,6 +163,22 @@ The `cephObjectStores` array in the values file will define a list of CephObject | `ingress.tls` | Ingress tls | `/` | | `ingress.ingressClassName` | Ingress tls | `""` | +### **Ceph Object Multisite** + +The `cephObjectRealms`, `cephObjectZoneGroups`, and `cephObjectZones` arrays in the values file define +[RGW multisite](../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) resources. They are +commented out by default; uncomment and adjust them to deploy a multisite configuration, then point a +`cephObjectStores` entry at the zone via its `spec.zone.name`. + +| Parameter | Description | Default | +| --------- | ----------- | ------- | +| `cephObjectRealms[].name` | The name of the CephObjectRealm | `realm-a` | +| `cephObjectRealms[].spec` | The CephObjectRealm spec, see the [CephObjectRealm CRD](../CRDs/Object-Storage/ceph-object-realm-crd.md) documentation. Omit to create a new realm; set `pull.endpoint` to sync a realm from another cluster. | not set | +| `cephObjectZoneGroups[].name` | The name of the CephObjectZoneGroup | `zonegroup-a` | +| `cephObjectZoneGroups[].spec` | The CephObjectZoneGroup spec, see the [CephObjectZoneGroup CRD](../CRDs/Object-Storage/ceph-object-zonegroup-crd.md) documentation. | see values.yaml | +| `cephObjectZones[].name` | The name of the CephObjectZone | `zone-a` | +| `cephObjectZones[].spec` | The CephObjectZone spec, see the [CephObjectZone CRD](../CRDs/Object-Storage/ceph-object-zone-crd.md) documentation. | see values.yaml | + ### **Existing Clusters** If you have an existing CephCluster CR that was created without the helm chart and you want the helm diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml new file mode 100644 index 000000000..639f0e9b5 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectrealm.yaml @@ -0,0 +1,13 @@ +{{- $root := . -}} +{{- range $realm := .Values.cephObjectRealms }} +--- +kind: CephObjectRealm +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $realm.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +{{- with $realm.spec }} +spec: + {{- . | toYaml | nindent 2 }} +{{- end }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml new file mode 100644 index 000000000..52639eaf5 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectzone.yaml @@ -0,0 +1,11 @@ +{{- $root := . -}} +{{- range $zone := .Values.cephObjectZones }} +--- +kind: CephObjectZone +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $zone.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +spec: + {{- $zone.spec | toYaml | nindent 2 }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml b/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml new file mode 100644 index 000000000..cec226f48 --- /dev/null +++ b/deploy/charts/rook-ceph-cluster/templates/cephobjectzonegroup.yaml @@ -0,0 +1,11 @@ +{{- $root := . -}} +{{- range $zonegroup := .Values.cephObjectZoneGroups }} +--- +kind: CephObjectZoneGroup +apiVersion: ceph.rook.io/v1 +metadata: + name: {{ $zonegroup.name }} + namespace: {{ $root.Release.Namespace }} # namespace:cluster +spec: + {{- $zonegroup.spec | toYaml | nindent 2 }} +{{- end }} diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index d3f530512..7785748e1 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -698,6 +698,43 @@ cephObjectStores: # - name: internal # namespace: kube-system # sectionName: https +## CephObjectRealm, CephObjectZoneGroup, and CephObjectZone configure RGW multisite. They are +## disabled by default; remove the comments and set desired values to enable them. See +## https://rook.io/docs/rook/latest/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/ +## The example below creates a new realm on this (primary) cluster. On a secondary cluster that +## syncs from another realm, add a `spec.pull.endpoint` to the realm instead (see the realm CRD docs). +#cephObjectRealms: +# - name: realm-a +# # spec is only required to pull a realm from another cluster; omit it to create a new realm +# # spec: +# # pull: +# # endpoint: http:// +#cephObjectZoneGroups: +# - name: zonegroup-a +# spec: +# realm: realm-a +#cephObjectZones: +# - name: zone-a +# spec: +# zoneGroup: zonegroup-a +# metadataPool: +# failureDomain: host +# replicated: +# size: 3 +# requireSafeReplicaSize: true +# dataPool: +# failureDomain: host +# replicated: +# size: 3 +# requireSafeReplicaSize: true +# parameters: +# compression_mode: none +# bulk: "true" +# # recommended to set this value if ingress used for exposing rgw endpoints +# # customEndpoints: +# # - "http://rgw-a.fqdn" +# # if the pools need to be removed from backend enable following setting +# # preservePoolsOnDelete: false ## cephECBlockPools are disabled by default, please remove the comments and set desired values to enable it ## For erasure coded a replicated metadata pool is required. ## https://rook.io/docs/rook/latest/CRDs/Shared-Filesystem/ceph-filesystem-crd/#erasure-coded diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 6be6bb6d0..051241a1e 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -285,6 +285,49 @@ func (h *CephInstaller) removeCephClusterHelmResources() { if !h.k8shelper.WaitUntilPodWithLabelDeleted(fmt.Sprintf("rook_object_store=%s", ObjectStoreName), h.settings.Namespace) { assert.Fail(h.T(), "rgw did not stop via helm uninstall") } + + if h.settings.UseMultisiteObjectStore { + h.removeMultisiteHelmResources(ObjectStoreName) + } +} + +// removeMultisiteHelmResources tears down the CephObjectRealm, CephObjectZoneGroup, +// CephObjectZone, and shared rgw pools that CreateObjectStoreConfiguration added +// to the chart. They are not part of the chart's default storage CRs, so they +// must be removed explicitly or they orphan and trip the teardown pool check. +// Order matters: the zone's deletion finalizer runs radosgw-admin against realm +// metadata stored in the .rgw.root pool, so the store (already deleted above), +// zone, zone group, and realm must all be gone before their pools are removed. +func (h *CephInstaller) removeMultisiteHelmResources(name string) { + ctx := context.TODO() + ceph := h.k8shelper.RookClientset.CephV1() + ns := h.settings.Namespace + + deleteAndWait := func(kind, crName string, del func() error, get func() error) { + if err := del(); err != nil { + assert.True(h.T(), kerrors.IsNotFound(err), "unexpected error deleting %s %q", kind, crName) + return + } + if err := h.k8shelper.WaitForCustomResourceDeletion(ns, crName, get); err != nil { + assert.NoError(h.T(), err, "failed waiting for %s %q deletion", kind, crName) + } + } + + deleteAndWait("CephObjectZone", name, + func() error { return ceph.CephObjectZones(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZones(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectZoneGroup", name, + func() error { return ceph.CephObjectZoneGroups(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectZoneGroups(ns).Get(ctx, name, v1.GetOptions{}); return err }) + deleteAndWait("CephObjectRealm", name, + func() error { return ceph.CephObjectRealms(ns).Delete(ctx, name, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephObjectRealms(ns).Get(ctx, name, v1.GetOptions{}); return err }) + + for poolName := range rgwSharedPools(name) { + deleteAndWait("CephBlockPool", poolName, + func() error { return ceph.CephBlockPools(ns).Delete(ctx, poolName, v1.DeleteOptions{}) }, + func() error { _, err := ceph.CephBlockPools(ns).Get(ctx, poolName, v1.GetOptions{}); return err }) + } } // ConfirmHelmClusterInstalledCorrectly runs some validation to check whether the helm chart installed correctly. @@ -381,8 +424,32 @@ func (h *CephInstaller) CreateFileSystemConfiguration(values map[string]interfac return nil } -// CreateObjectStoreConfiguration creates an object store configuration +// rgwSharedPools maps the CephBlockPool CR name to its rados pool name (empty +// means the operator derives the pool name from the CR name) for the multisite +// object store configured by CreateObjectStoreConfiguration. .rgw.root is the +// realm-global metadata pool; the operator sets pg_num_min=8 on it, so it must +// be created with pg_num>=8 or the zone reconcile fails. +func rgwSharedPools(name string) map[string]string { + return map[string]string{ + "rgw.root": ".rgw.root", + name + ".rgw.control": "", + name + ".rgw.meta": "", + name + ".rgw.log": "", + name + ".rgw.otp": "", + name + ".rgw.buckets.index": "", + name + ".rgw.buckets.data": "", + name + ".rgw.buckets.data.foo": "", + } +} + +// CreateObjectStoreConfiguration adds the chart's object store and its bucket +// storage class to the Helm values. When UseMultisiteObjectStore is set it +// configures an RGW multisite zone; otherwise it configures a plain store. func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + if h.settings.UseMultisiteObjectStore { + return h.createMultisiteObjectStoreConfiguration(values, name, scName) + } + testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false, false)) var testObjectStoreCRD map[string]interface{} if err := yaml.Unmarshal(testObjectStoreBytes, &testObjectStoreCRD); err != nil { @@ -409,3 +476,112 @@ func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interfa } return nil } + +// createMultisiteObjectStoreConfiguration configures the object store as an RGW +// multisite zone, exercising the chart's CephObjectRealm, CephObjectZoneGroup, +// and CephObjectZone templates on install. The realm/zone group/zone/shared-pools +// layout mirrors the fixture in tests/integration/object/util/sharedstore. +func (h *CephInstaller) createMultisiteObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { + poolNames := rgwSharedPools(name) + rgwPools := make([]map[string]interface{}, 0, len(poolNames)) + for poolName, radosName := range poolNames { + pgNum := "1" + if radosName == ".rgw.root" { + pgNum = "8" + } + spec := map[string]interface{}{ + "replicated": map[string]interface{}{ + "size": 1, + "requireSafeReplicaSize": false, + }, + "parameters": map[string]interface{}{ + "pg_autoscale_mode": "off", + "pg_num": pgNum, + }, + } + // spec.name is enum-validated (.rgw.root/.nfs/.mgr) and must be omitted + // rather than set to "" when the pool name matches the CR name. + if radosName != "" { + spec["name"] = radosName + } + rgwPools = append(rgwPools, map[string]interface{}{ + "name": poolName, + "spec": spec, + "storageClass": map[string]interface{}{ + "enabled": false, + }, + }) + } + + // Append the rgw pools to the block pools already configured for CSI so both + // sets land in the chart's cephBlockPools list. + existingPools, _ := values["cephBlockPools"].([]map[string]interface{}) + values["cephBlockPools"] = append(existingPools, rgwPools...) + + values["cephObjectRealms"] = []map[string]interface{}{ + { + "name": name, + }, + } + values["cephObjectZoneGroups"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "realm": name, + }, + }, + } + values["cephObjectZones"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zoneGroup": name, + "sharedPools": map[string]interface{}{ + "poolPlacements": []map[string]interface{}{ + { + "name": "default", + "default": true, + "metadataPoolName": name + ".rgw.buckets.index", + "dataPoolName": name + ".rgw.buckets.data", + "storageClasses": []map[string]interface{}{ + { + "name": "FOO", + "dataPoolName": name + ".rgw.buckets.data.foo", + }, + }, + }, + }, + }, + }, + }, + } + + storageClassBytes := []byte(h.Manifests.GetBucketStorageClass(name, scName, "Delete")) + var testObjectStoreSC map[string]interface{} + if err := yaml.Unmarshal(storageClassBytes, &testObjectStoreSC); err != nil { + return err + } + + values["cephObjectStores"] = []map[string]interface{}{ + { + "name": name, + "spec": map[string]interface{}{ + "zone": map[string]interface{}{ + "name": name, + }, + "gateway": map[string]interface{}{ + "port": 8080, + "instances": 2, + "resources": nil, + }, + }, + "storageClass": map[string]interface{}{ + "enabled": true, + "name": scName, + "parameters": testObjectStoreSC["parameters"], + "reclaimPolicy": "Delete", + }, + }, + } + return nil +} diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index 5383f557b..090c542f7 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -54,6 +54,12 @@ type TestCephSettings struct { KubernetesVersion string EnableCsiOperator bool ClusterConcurrency int + + // UseMultisiteObjectStore configures the Helm chart's object store as an RGW + // multisite zone (realm/zone group/zone/shared pools) instead of a plain + // store. Only valid for local-chart installs, which carry the multisite + // templates, so it must stay off for the upgrade suite's older base chart. + UseMultisiteObjectStore bool } func (s *TestCephSettings) ApplyEnvVars() { diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 360368fec..4a9a342bc 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -59,19 +59,20 @@ type HelmSuite struct { func (h *HelmSuite) SetupSuite() { namespace := "helm-ns" h.settings = &installer.TestCephSettings{ - Namespace: namespace, - OperatorNamespace: namespace, - StorageClassName: "", - UseHelm: true, - UsePVC: false, - Mons: 1, - SkipOSDCreation: false, - EnableDiscovery: true, - ChangeHostName: true, - ConnectionsEncrypted: true, - RookVersion: installer.LocalBuildTag, - CephVersion: installer.SquidVersion, - EnableCsiOperator: true, + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UseMultisiteObjectStore: true, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + EnableDiscovery: true, + ChangeHostName: true, + ConnectionsEncrypted: true, + RookVersion: installer.LocalBuildTag, + CephVersion: installer.SquidVersion, + EnableCsiOperator: true, } h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings)