From edcedd31245809aae50e3aae5448d2f76511e6e1 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Tue, 30 Jul 2024 11:24:57 +0200 Subject: [PATCH] docs: pool placement design doc Signed-off-by: Artem Torubarov --- design/ceph/object/store.md | 286 ++++++++++++++++++++++++++++++++++++ 1 file changed, 286 insertions(+) diff --git a/design/ceph/object/store.md b/design/ceph/object/store.md index 2bfcf442a..261e5d655 100644 --- a/design/ceph/object/store.md +++ b/design/ceph/object/store.md @@ -221,6 +221,292 @@ spec: name: zone1 ``` +### Pool Placement targets and Storage Classes + +Object Storage API allows users to specify where bucket data will be stored during bucket creation. With `` parameter in S3 API and `X-Storage-Policy` header in SWIFT. Similarly, users can override where object data will be stored by setting `X-Amz-Storage-Class` and `X-Object-Storage-Class` during object creation. + +In Ceph these data locations represented by Pools. Placement targets control which Pools are associated with a particular bucket. A bucket’s placement target is selected on creation, and cannot be modified (See [Ceph doc](https://docs.ceph.com/en/latest/radosgw/placement/#pool-placement-and-storage-classes)). Placement target can also define custom Storage classes to override data pool where object will be stored. + +Ceph administrator creates set of `index_pool`, `data_pool`, and `data_extra_pool` per each target placement. Select arbitrary names for target placements. For example `hot` and `cold`: +> [!NOTE] +> `data_extra_pool` is for data that cannot use erasure coding. For example, multi-part uploads allow uploading a large object such as a movie in multiple parts. These parts must first be stored without erasure coding. So if `data_pool` is **not** erasure coded, then there is not need for `data_extra_pool`. + +```yaml +# Default Data pool +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-meta-pool + namespace: rook-ceph +spec: + ... +--- +# Default index pool +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-data-pool + namespace: rook-ceph +spec: + ... +# Bucket Data pool for hot storage +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-hot-data-pool + namespace: rook-ceph +spec: + ... +--- +# Bucket index pool for hot storage +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-hot-meta-pool + namespace: rook-ceph +spec: + ... +--- +# Bucket Data pool for cold storage +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-cold-data-pool + namespace: rook-ceph +spec: + ... +--- +# Bucket index pool for cold storage +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: rgw-cold-meta-pool + namespace: rook-ceph +spec: + ... +``` + +If needed, create additional pool to use it in custom StorageClass, for example `GLACIER` and `REDUCED_REDUNDANCY`: +> [!NOTE] +> Ceph allows arbitrary name for StorageClasses, however some clients/libs insist on AWS names so it is recommended to use one of the [valid x-amz-storage-class values](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#API_PutObject_RequestSyntax) for better compatibility: +> +> `STANDARD | REDUCED_REDUNDANCY | STANDARD_IA | ONEZONE_IA | INTELLIGENT_TIERING | GLACIER | DEEP_ARCHIVE | OUTPOSTS | GLACIER_IR | SNOW | EXPRESS_ONEZONE` + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: glacier-data-pool + namespace: rook-ceph +spec: + ... +--- +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: reduced-data-pool + namespace: rook-ceph +spec: + ... +``` + +Then update `CephObjectStore` CRD `sharedPools` section with target placements and StorageClasses definitions: + +```yaml +spec: + # [OPTIONAL] - sharedPools is mutually exclusive with spec.metadataPool and spec.dataPool + # It is the place to tell RGW which pools can be used to store buckets and objects data and metadata. + # All pools under this section should be created beforehand. + # Rook will append rados namespace to each of the pools, so it is safe to use the same pool across multiple RGW instances. + # It is also safe to use the same pool in different target placements. + sharedPools: + # [EXISTING,OPTIONAL] - metadata pool. Will be used by default to store bucket index + # default poolPlacement.metadataPoolName will override this value, so it is not makes sense to use together with poolPlacements. + metadataPoolName: rgw-meta-pool # results to rgw-meta-pool:.index + + # [EXISTING,OPTIONAL] - data pool. Will be used by default to store bucket objects + # default poolPlacement.dataPoolName will override this value, so it is not makes sense to use together with poolPlacements. + dataPoolName: rgw-data-pool # results to rgw-data-pool:.data + + # New settings: + # [OPTIONAL] - defines pool placement targets. + # Placement targets control which Pools are associated with a particular bucket. + # See: https://docs.ceph.com/en/latest/radosgw/placement/#placement-targets + poolPlacements: + # Default pool placement: + - name: "default" # [REQUIRED] - placement id. Placement with name "default" will be used as default, + # and therefore will override sharedPools.metadataPoolName and sharedPools.dataPoolName if presented. + # This means that default poolPlacement is also mutually exclusive with spec.metadataPool and spec.dataPool + metadataPoolName: rgw-meta-pool # [REQUIRED] results to rgw-meta-pool:.index + dataPoolName: rgw-data-pool # [REQUIRED] results to rgw-data-pool:.data + dataNonECPoolName: "" # [OPTIONAL] results to rgw-data-pool:.data.nonec + # [OPTIONAL] - Storage classes for cold placement target. + storageClasses: + - name: GLACIER # [REQUIRED] StorageClass name. Can be arbitrary but it is better to use one of AWS x-amz-storage-class names. + dataPoolName: glacier-data-pool # [REQUIRED] results to glacier-data-pool:.glacier + # Pool placement target named "cold" + - name: "cold" # [REQUIRED] - placement id. Can be arbitrary. + metadataPoolName: rgw-cold-meta-pool # [REQUIRED] results to rgw-cold-meta-pool:.cold.index + dataPoolName: rgw-cold-data-pool # [REQUIRED] results to rgw-cold-data-pool:.cold.data + dataNonECPoolName: "" # [OPTIONAL] results to rgw-cold-data-pool:.cold.data.nonec + # [OPTIONAL] - Storage classes for cold placement target. + storageClasses: + - name: GLACIER # [REQUIRED] StorageClass name. Can be arbitrary but it is better to use one of AWS x-amz-storage-class names. + dataPoolName: glacier-data-pool # [REQUIRED] results to glacier-data-pool:.glacier + # Pool placement target named "hot" + - name: "hot" + metadataPoolName: rgw-hot-meta-pool # [REQUIRED] results to rgw-hot-meta-pool:.hot.index + dataPoolName: rgw-hot-data-pool # [REQUIRED] results to rgw-hot-data-pool:.hot.data + dataNonECPoolName: "" # [OPTIONAL] results to rgw-hot-data-pool:.hot.data.nonec + # [OPTIONAL] - Storage classes for hot placement target. + storageClasses: + - name: REDUCED_REDUNDANCY # [REQUIRED] StorageClass name + dataPoolName: reduced-data-pool # [REQUIRED] results to rgw-data-reduce-pool:.reduced_redundancy +``` + +Here we specified that pools `rgw-meta-pool` and `rgw-data-pool` will be used by default and StorageClass `GLACIER` can be used to override object data pool to `glacier-data-pool`. +Then we also defined 2 additional bucket target placements: `hot` and `cold` along with their StorageClasses. + +Rook CephObjectStore controller reconciliation loop for target placements: + +1. CephObjectStore CRD changed +2. Validate `spec.sharedPools`: +3. Check that listed pools are created. +4. Calculate zone and zonegroup name: `CephObjectStore.spec.zone.name | CephObjectStore.metadata.name` - equal to zone name for multisite, otherwise equal to rgw name. +4. get default zonegroup: `radosgw-admin zonegroup get --rgw-zonegroup=` + ```json + { + "placement_targets": [ + { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "STANDARD" + ] + } + ], + "default_placement": "default-placement", + ... + } + ``` +5. update json to: + ```json + { + "placement_targets": [ + { + "name": "default-placement", + "tags": [], + "storage_classes": [ + "STANDARD","GLACIER" + ] + } + { + "name": "hot", + "tags": [], + "storage_classes": [ + "STANDARD","GLACIER" + ] + }, + { + "name": "cold", + "tags": [], + "storage_classes": [ + "STANDARD","REDUCED_REDUNDANCY" + ] + } + ], + "default_placement": "default-placement", + ... + } + ``` +6. update with `radosgw-admin zonegroup set --rgw-zonegroup= --infile zonegroup.json` +7. get default zone: `radosgw-admin zone get --rgw-zone=` + ```json + { + "placement_pools": [ + { + "key": "default-placement", + "val": { + "index_pool": ".rgw.buckets.index", + "storage_classes": { + "STANDARD": { + "data_pool": ".rgw.buckets.data" + } + }, + "data_extra_pool": ".rgw.buckets.non-ec", + "index_type": 0, + "inline_data": true + } + } + ], + ... + } + ``` +8. update json to + ```json + { + "placement_pools": [ + { + "key": "default-placement", + "val": { + "index_pool": "rgw-meta-pool:.index", + "storage_classes": { + "STANDARD": { + "data_pool": "rgw-data-pool:.data" + }, + "GLACIER": { + "data_pool": "glacier-data-pool:.glacier" + } + + }, + "data_extra_pool": "rgw-data-pool:.data.nonec", + "index_type": 0, + "inline_data": true + } + }, + { + "key": "cold", + "val": { + "index_pool": "rgw-cold-meta-pool:.cold.index", + "storage_classes": { + "STANDARD": { + "data_pool": "rgw-cold-data-pool:.cold.data" + }, + "GLACIER": { + "data_pool": "glacier-data-pool:.glacier" + } + + }, + "data_extra_pool": "rgw-cold-data-pool:.cold.data.nonec", + "index_type": 0, + "inline_data": true + } + }, + { + "key": "hot", + "val": { + "index_pool": "rgw-cold-meta-pool:.hot.index", + "storage_classes": { + "STANDARD": { + "data_pool": "rgw-hot-data-pool:.hot.data" + }, + "REDUCED_REDUNDANCY": { + "data_pool": "rgw-data-reduce-pool:.reduced_redundancy" + } + + }, + "data_extra_pool": "rgw-hot-data-pool:.hot.data.nonec", + "index_type": 0, + "inline_data": true + } + } + ], + ... + } + ``` +9. update with `radosgw-admin zone set --rgw-zone= --infile zone.json` +10. commit: `radosgw-admin period update --commit` + ### Gateway The gateway settings correspond to the RGW service.